class Settings(SingletonConfigurable): """ Stores the settings for the registry """ DEFAULT_EXTENSION_TYPES = [ "yuuno.vs.extension.VapourSynth", ] registry_type: str = DottedObjectName("yuuno.core.registry.Registry", config=True) registry = Any() extension_types = List(Union([DottedObjectName(), Type()]), config=True) @observe('registry_type') def _reset_registry_on_reset(self, change: dict) -> None: self.registry = import_item(change['new'])() @default('registry') def _auto_registry(self) -> All: return import_item(self.registry_type)() @default('extension_types') def _auto_extension_types(self): return self.DEFAULT_EXTENSION_TYPES + list(discover_extensions())
class QtKernelManager(KernelManager, QtKernelManagerMixin): """A KernelManager with Qt signals for restart""" client_class = DottedObjectName('qtconsole.client.QtKernelClient') autorestart = Bool(True, config=True) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._is_restarting = False def start_restarter(self): if self.autorestart and self.has_kernel: if self._restarter is None: self._restarter = QtKernelRestarter( kernel_manager=self, parent=self, log=self.log, ) self._is_restarting = True self._restarter.start() def stop_restarter(self): if self.autorestart: if self._restarter is not None: self._restarter.stop() def post_start_kernel(self, **kw): super().post_start_kernel(**kw) if self._is_restarting: self.kernel_restarted.emit() self._is_restarting = False
class NbcatApp(NbConvertApp): # Override CLI argument and stuff name = 'nbcat' aliases = app_aliases flags = app_flags # Use nbconvert's config file @default('config_file_name') def _default_config_file_name(self): return 'jupyter_nbconvert_config' # Override CLI documentation # Indentation gets stripped here description = Unicode(''' View contents of notebook files (*.ipynb) in the terminal. Extension to the "jupyter nbconvert" application. New options are "--256colors", "--style", and "--list-styles". Other options are identical. ''') examples = None # Override export format to always use terminal export_format = Unicode('terminal') # Use stdout by default writer_class = DottedObjectName( 'StdoutWriter', help=NbConvertApp.writer_class.help, ).tag(config=True) # Seems to be the list of Configurable classes used the the application @property def classes(self): return super().classes + [NbcatApp, TerminalExporter] # Triggers listing of Pygments styles instead of converting list_styles = Bool( False, config=True, help='List available syntax highlighting styles.', ) def print_styles_list(self): """Print list of Pygments styles.""" # print(*sorted(get_all_styles()), sep='\n') # Run the corresponding command from pygment's CLI, also prints descriptions from pygments.cmdline import main main(['pygmentize', '-L', 'styles']) def start(self): # If list_styles flag set, print styles list and quit if self.list_styles: self.print_styles_list() raise NoStart() super().start()
class InProcessKernelManager(KernelManager): """A manager for an in-process kernel. This class implements the interface of `jupyter_client.kernelmanagerabc.KernelManagerABC` and allows (asynchronous) frontends to be used seamlessly with an in-process kernel. See `jupyter_client.kernelmanager.KernelManager` for docstrings. """ # The kernel process with which the KernelManager is communicating. kernel = Instance('ipykernel.inprocess.ipkernel.InProcessKernel', allow_none=True) # the client class for KM.client() shortcut client_class = DottedObjectName('ipykernel.inprocess.BlockingInProcessKernelClient') def _blocking_class_default(self): from .blocking import BlockingInProcessKernelClient return BlockingInProcessKernelClient def _session_default(self): # don't sign in-process messages return Session(key=b'', parent=self) #-------------------------------------------------------------------------- # Kernel management methods #-------------------------------------------------------------------------- def start_kernel(self, **kwds): from ipykernel.inprocess.ipkernel import InProcessKernel self.kernel = InProcessKernel(parent=self, session=self.session) def shutdown_kernel(self): self._kill_kernel() def restart_kernel(self, now=False, **kwds): self.shutdown_kernel() self.start_kernel(**kwds) @property def has_kernel(self): return self.kernel is not None def _kill_kernel(self): self.kernel = None def interrupt_kernel(self): raise NotImplementedError("Cannot interrupt in-process kernel.") def signal_kernel(self, signum): raise NotImplementedError("Cannot signal in-process kernel.") def is_alive(self): return self.kernel is not None def client(self, **kwargs): kwargs['kernel'] = self.kernel return super(InProcessKernelManager, self).client(**kwargs)
class IPythonVapoursynthExtension(Extension): """ This extension implements VapourSynth-specific features for IPython. """ _name = "ipy_vs" feature_classes: Listing[str] = List(DottedObjectName(), default_value=[ "yuuno_ipython.ipy_vs.log.LogWriterFeature", "yuuno_ipython.ipy_vs.encode.Encode", "yuuno_ipython.ipy_vs.runvpy.RunVPy", "yuuno_ipython.ipy_vs.vsscript.Use_VSScript", "yuuno_ipython.ipy_vs.stateful_editor.StatefulEditorFeature" ], config=True, help="List of additional features to load.") features: Listing[Feature] = List(Instance(Feature)) yuuno: Yuuno = Instance(Yuuno) @default("features") def _default_features(self): return [] @default("yuuno") def _default_yuuno(self): return Yuuno.instance() @classmethod def is_supported(cls): try: import IPython except ImportError: return False from yuuno_ipython.ipython.environment import YuunoIPythonEnvironment if not isinstance(Yuuno.instance().environment, YuunoIPythonEnvironment): return False from yuuno.vs.extension import VapourSynth return VapourSynth.is_supported() def initialize(self): for feature in self.feature_classes: self.yuuno.log.debug(f"Loading feature: {feature}") feature = import_item(feature) feature_inst = feature(extension=self) self.features.append(feature_inst) feature_inst.initialize() def deinitialize(self): for feature in self.features[:]: feature.deinitialize() self.features.remove(feature)
class AsyncMultiKernelManager(MultiKernelManager): kernel_manager_class = DottedObjectName( "jupyter_client.ioloop.AsyncIOLoopKernelManager", config=True, help="""The kernel manager class. This is configurable to allow subclassing of the AsyncKernelManager for customized behavior. """, ) start_kernel = MultiKernelManager._async_start_kernel shutdown_kernel = MultiKernelManager._async_shutdown_kernel shutdown_all = MultiKernelManager._async_shutdown_all
class SpyderKernelApp(IPKernelApp): outstream_class = DottedObjectName( 'spyder_kernels.console.outstream.TTYOutStream') def init_pdb(self): """ This method was added in IPykernel 5.3.1 and it replaces the debugger used by the kernel with a new class introduced in IPython 7.15 during kernel's initialization. Therefore, it doesn't allow us to use our debugger. """ pass
class AsyncKernelManager(KernelManager): # the class to create with our `client` method client_class: DottedObjectName = DottedObjectName( "jupyter_client.asynchronous.AsyncKernelClient" ) client_factory: Type = Type(klass="jupyter_client.asynchronous.AsyncKernelClient") _launch_kernel = KernelManager._async_launch_kernel start_kernel = KernelManager._async_start_kernel finish_shutdown = KernelManager._async_finish_shutdown shutdown_kernel = KernelManager._async_shutdown_kernel restart_kernel = KernelManager._async_restart_kernel _send_kernel_sigterm = KernelManager._async_send_kernel_sigterm _kill_kernel = KernelManager._async_kill_kernel interrupt_kernel = KernelManager._async_interrupt_kernel signal_kernel = KernelManager._async_signal_kernel is_alive = KernelManager._async_is_alive
class AsyncMultiKernelManager(MultiKernelManager): kernel_manager_class = DottedObjectName( "jupyter_client.ioloop.AsyncIOLoopKernelManager", config=True, help="""The kernel manager class. This is configurable to allow subclassing of the AsyncKernelManager for customized behavior. """, ) use_pending_kernels = Bool( False, help= """Whether to make kernels available before the process has started. The kernel has a `.ready` future which can be awaited before connecting""", ).tag(config=True) start_kernel = MultiKernelManager._async_start_kernel shutdown_kernel = MultiKernelManager._async_shutdown_kernel shutdown_all = MultiKernelManager._async_shutdown_all
class QtKernelManager(KernelManager, QtKernelManagerMixin): """A KernelManager with Qt signals for restart""" client_class = DottedObjectName('qtconsole.client.QtKernelClient') autorestart = Bool(True, config=True) def start_restarter(self): if self.autorestart and self.has_kernel: if self._restarter is None: self._restarter = QtKernelRestarter( kernel_manager=self, parent=self, log=self.log, ) self._restarter.add_callback(self._handle_kernel_restarted) self._restarter.start() def stop_restarter(self): if self.autorestart: if self._restarter is not None: self._restarter.stop() def _handle_kernel_restarted(self): self.kernel_restarted.emit()
class AsyncMultiKernelManager(MultiKernelManager): kernel_manager_class = DottedObjectName( "jupyter_client.ioloop.AsyncIOLoopKernelManager", config=True, help="""The kernel manager class. This is configurable to allow subclassing of the AsyncKernelManager for customized behavior. """ ) _starting_kernels = Dict() async def _add_kernel_when_ready(self, kernel_id, km, kernel_awaitable): await kernel_awaitable self._kernels[kernel_id] = km async def start_kernel(self, kernel_name=None, **kwargs): """Start a new kernel. The caller can pick a kernel_id by passing one in as a keyword arg, otherwise one will be generated using new_kernel_id(). The kernel ID for the newly started kernel is returned. """ km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) if not isinstance(km, AsyncKernelManager): self.log.warning("Kernel manager class ({km_class}) is not an instance of 'AsyncKernelManager'!". format(km_class=self.kernel_manager_class.__class__)) fut = asyncio.ensure_future( self._add_kernel_when_ready( kernel_id, km, km.start_kernel(**kwargs) ) ) self._starting_kernels[kernel_id] = fut await fut del self._starting_kernels[kernel_id] return kernel_id async def shutdown_kernel(self, kernel_id, now=False, restart=False): """Shutdown a kernel by its kernel uuid. Parameters ========== kernel_id : uuid The id of the kernel to shutdown. now : bool Should the kernel be shutdown forcibly using a signal. restart : bool Will the kernel be restarted? """ self.log.info("Kernel shutdown: %s" % kernel_id) km = self.get_kernel(kernel_id) ports = ( km.shell_port, km.iopub_port, km.stdin_port, km.hb_port, km.control_port ) await km.shutdown_kernel(now, restart) self.remove_kernel(kernel_id) if km.cache_ports and not restart: for port in ports: self.currently_used_ports.remove(port) async def finish_shutdown(self, kernel_id, waittime=None, pollinterval=0.1): """Wait for a kernel to finish shutting down, and kill it if it doesn't """ km = self.get_kernel(kernel_id) await km.finish_shutdown(waittime, pollinterval) self.log.info("Kernel shutdown: %s" % kernel_id) async def interrupt_kernel(self, kernel_id): """Interrupt (SIGINT) the kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ km = self.get_kernel(kernel_id) await km.interrupt_kernel() self.log.info("Kernel interrupted: %s" % kernel_id) async def signal_kernel(self, kernel_id, signum): """Sends a signal to the kernel by its uuid. Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. Parameters ========== kernel_id : uuid The id of the kernel to signal. """ km = self.get_kernel(kernel_id) await km.signal_kernel(signum) self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum)) async def restart_kernel(self, kernel_id, now=False): """Restart a kernel by its uuid, keeping the same ports. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ km = self.get_kernel(kernel_id) await km.restart_kernel(now) self.log.info("Kernel restarted: %s" % kernel_id) async def _shutdown_starting_kernel(self, kid, now): if kid in self._starting_kernels: await self._starting_kernels[kid] await self.shutdown_kernel(kid, now=now) async def shutdown_all(self, now=False): """Shutdown all kernels.""" kids = self.list_kernel_ids() futs = [self.shutdown_kernel(kid, now=now) for kid in kids] futs += [ self._shutdown_starting_kernel(kid, now=now) for kid in self._starting_kernels.keys() ] await asyncio.gather(*futs)
class NbConvertApp(JupyterApp): """Application used to convert from notebook file type (``*.ipynb``)""" version = __version__ name = 'jupyter-nbconvert' aliases = nbconvert_aliases flags = nbconvert_flags @default('log_level') def _log_level_default(self): return logging.INFO classes = List() @default('classes') def _classes_default(self): classes = [NbConvertBase] for pkg in (exporters, preprocessors, writers, postprocessors): for name in dir(pkg): cls = getattr(pkg, name) if isinstance(cls, type) and issubclass(cls, Configurable): classes.append(cls) return classes description = Unicode( u"""This application is used to convert notebook files (*.ipynb) to various other formats. WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""") output_base = Unicode('', help='''overwrite base name use for output files. can only be used when converting one notebook at a time. ''').tag(config=True) use_output_suffix = Bool( True, help="""Whether to apply a suffix prior to the extension (only relevant when converting to notebook format). The suffix is determined by the exporter, and is usually '.nbconvert'.""").tag(config=True) output_files_dir = Unicode( '{notebook_name}_files', help='''Directory to copy extra files (figures) to. '{notebook_name}' in the string will be converted to notebook basename''').tag(config=True) examples = Unicode(u""" The simplest way to use nbconvert is > jupyter nbconvert mynotebook.ipynb which will convert mynotebook.ipynb to the default format (probably HTML). You can specify the export format with `--to`. Options include {0} > jupyter nbconvert --to latex mynotebook.ipynb Both HTML and LaTeX support multiple output templates. LaTeX includes 'base', 'article' and 'report'. HTML includes 'basic' and 'full'. You can specify the flavor of the format used. > jupyter nbconvert --to html --template basic mynotebook.ipynb You can also pipe the output to stdout, rather than a file > jupyter nbconvert mynotebook.ipynb --stdout PDF is generated via latex > jupyter nbconvert mynotebook.ipynb --to pdf You can get (and serve) a Reveal.js-powered slideshow > jupyter nbconvert myslides.ipynb --to slides --post serve Multiple notebooks can be given at the command line in a couple of different ways: > jupyter nbconvert notebook*.ipynb > jupyter nbconvert notebook1.ipynb notebook2.ipynb or you can specify the notebooks list in a config file, containing:: c.NbConvertApp.notebooks = ["my_notebook.ipynb"] > jupyter nbconvert --config mycfg.py """.format(get_export_names())) # Writer specific variables writer = Instance('nbconvert.writers.base.WriterBase', help="""Instance of the writer class used to write the results of the conversion.""", allow_none=True) writer_class = DottedObjectName('FilesWriter', help="""Writer class used to write the results of the conversion""").tag( config=True) writer_aliases = { 'fileswriter': 'nbconvert.writers.files.FilesWriter', 'debugwriter': 'nbconvert.writers.debug.DebugWriter', 'stdoutwriter': 'nbconvert.writers.stdout.StdoutWriter' } writer_factory = Type(allow_none=True) @observe('writer_class') def _writer_class_changed(self, change): new = change['new'] if new.lower() in self.writer_aliases: new = self.writer_aliases[new.lower()] self.writer_factory = import_item(new) # Post-processor specific variables postprocessor = Instance( 'nbconvert.postprocessors.base.PostProcessorBase', help="""Instance of the PostProcessor class used to write the results of the conversion.""", allow_none=True) postprocessor_class = DottedOrNone( help="""PostProcessor class used to write the results of the conversion""").tag( config=True) postprocessor_aliases = { 'serve': 'nbconvert.postprocessors.serve.ServePostProcessor' } postprocessor_factory = Type(None, allow_none=True) @observe('postprocessor_class') def _postprocessor_class_changed(self, change): new = change['new'] if new.lower() in self.postprocessor_aliases: new = self.postprocessor_aliases[new.lower()] if new: self.postprocessor_factory = import_item(new) export_format = Unicode( 'html', allow_none=False, help="""The export format to be used, either one of the built-in formats, or a dotted object name that represents the import path for an `Exporter` class""").tag(config=True) notebooks = List([], help="""List of notebooks to convert. Wildcards are supported. Filenames passed positionally will be added to the list. """).tag(config=True) from_stdin = Bool( False, help="read a single notebook from stdin.").tag(config=True) @catch_config_error def initialize(self, argv=None): """Initialize application, notebooks, writer, and postprocessor""" self.init_syspath() super(NbConvertApp, self).initialize(argv) self.init_notebooks() self.init_writer() self.init_postprocessor() def init_syspath(self): """Add the cwd to the sys.path ($PYTHONPATH)""" sys.path.insert(0, os.getcwd()) def init_notebooks(self): """Construct the list of notebooks. If notebooks are passed on the command-line, they override (rather than add) notebooks specified in config files. Glob each notebook to replace notebook patterns with filenames. """ # Specifying notebooks on the command-line overrides (rather than # adds) the notebook list if self.extra_args: patterns = self.extra_args else: patterns = self.notebooks # Use glob to replace all the notebook patterns with filenames. filenames = [] for pattern in patterns: # Use glob to find matching filenames. Allow the user to convert # notebooks without having to type the extension. globbed_files = glob.glob(pattern) globbed_files.extend(glob.glob(pattern + '.ipynb')) if not globbed_files: self.log.warn("pattern %r matched no files", pattern) for filename in globbed_files: if not filename in filenames: filenames.append(filename) self.notebooks = filenames def init_writer(self): """Initialize the writer (which is stateless)""" self._writer_class_changed({'new': self.writer_class}) self.writer = self.writer_factory(parent=self) if hasattr(self.writer, 'build_directory') and self.writer.build_directory != '': self.use_output_suffix = False def init_postprocessor(self): """Initialize the postprocessor (which is stateless)""" self._postprocessor_class_changed({'new': self.postprocessor_class}) if self.postprocessor_factory: self.postprocessor = self.postprocessor_factory(parent=self) def start(self): """Run start after initialization process has completed""" super(NbConvertApp, self).start() self.convert_notebooks() def init_single_notebook_resources(self, notebook_filename): """Step 1: Initialize resources This initializes the resources dictionary for a single notebook. Returns ------- dict resources dictionary for a single notebook that MUST include the following keys: - config_dir: the location of the Jupyter config directory - unique_key: the notebook name - output_files_dir: a directory where output files (not including the notebook itself) should be saved """ basename = os.path.basename(notebook_filename) notebook_name = basename[:basename.rfind('.')] if self.output_base: # strip duplicate extension from output_base, to avoid Basename.ext.ext if getattr(self.exporter, 'file_extension', False): base, ext = os.path.splitext(self.output_base) if ext == self.exporter.file_extension: self.output_base = base notebook_name = self.output_base self.log.debug("Notebook name is '%s'", notebook_name) # first initialize the resources we want to use resources = {} resources['config_dir'] = self.config_dir resources['unique_key'] = notebook_name output_files_dir = (self.output_files_dir.format( notebook_name=notebook_name)) resources['output_files_dir'] = output_files_dir return resources def export_single_notebook(self, notebook_filename, resources, input_buffer=None): """Step 2: Export the notebook Exports the notebook to a particular format according to the specified exporter. This function returns the output and (possibly modified) resources from the exporter. Parameters ---------- notebook_filename : str name of notebook file. resources : dict input_buffer : readable file-like object returning unicode. if not None, notebook_filename is ignored Returns ------- output dict resources (possibly modified) """ try: if input_buffer is not None: output, resources = self.exporter.from_file( input_buffer, resources=resources) else: output, resources = self.exporter.from_filename( notebook_filename, resources=resources) except ConversionException: self.log.error("Error while converting '%s'", notebook_filename, exc_info=True) self.exit(1) return output, resources def write_single_notebook(self, output, resources): """Step 3: Write the notebook to file This writes output from the exporter to file using the specified writer. It returns the results from the writer. Parameters ---------- output : resources : dict resources for a single notebook including name, config directory and directory to save output Returns ------- file results from the specified writer output of exporter """ if 'unique_key' not in resources: raise KeyError( "unique_key MUST be specified in the resources, but it is not") notebook_name = resources['unique_key'] if self.use_output_suffix and not self.output_base: notebook_name += resources.get('output_suffix', '') write_results = self.writer.write(output, resources, notebook_name=notebook_name) return write_results def postprocess_single_notebook(self, write_results): """Step 4: Post-process the written file Only used if a postprocessor has been specified. After the converted notebook is written to a file in Step 3, this post-processes the notebook. """ # Post-process if post processor has been defined. if hasattr(self, 'postprocessor') and self.postprocessor: self.postprocessor(write_results) def convert_single_notebook(self, notebook_filename, input_buffer=None): """Convert a single notebook. Performs the following steps: 1. Initialize notebook resources 2. Export the notebook to a particular format 3. Write the exported notebook to file 4. (Maybe) postprocess the written file Parameters ---------- notebook_filename : str input_buffer : If input_buffer is not None, conversion is done and the buffer is used as source into a file basenamed by the notebook_filename argument. """ if input_buffer is None: self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format) else: self.log.info("Converting notebook into %s", self.export_format) resources = self.init_single_notebook_resources(notebook_filename) output, resources = self.export_single_notebook( notebook_filename, resources, input_buffer=input_buffer) write_results = self.write_single_notebook(output, resources) self.postprocess_single_notebook(write_results) def convert_notebooks(self): """Convert the notebooks in the self.notebook traitlet """ # check that the output base isn't specified if there is more than # one notebook to convert if self.output_base != '' and len(self.notebooks) > 1: self.log.error(""" UsageError: --output flag or `NbConvertApp.output_base` config option cannot be used when converting multiple notebooks. """) self.exit(1) # initialize the exporter cls = get_exporter(self.export_format) self.exporter = cls(config=self.config) # no notebooks to convert! if len(self.notebooks) == 0 and not self.from_stdin: self.print_help() sys.exit(-1) # convert each notebook if not self.from_stdin: for notebook_filename in self.notebooks: self.convert_single_notebook(notebook_filename) else: input_buffer = unicode_stdin_stream() # default name when conversion from stdin self.convert_single_notebook("notebook.ipynb", input_buffer=input_buffer)
class MultiKernelManager(LoggingConfigurable): """A class for managing multiple kernels.""" default_kernel_name = Unicode(NATIVE_KERNEL_NAME, config=True, help="The name of the default kernel to start" ) kernel_spec_manager = Instance(KernelSpecManager, allow_none=True) kernel_manager_class = DottedObjectName( "jupyter_client.ioloop.IOLoopKernelManager", config=True, help="""The kernel manager class. This is configurable to allow subclassing of the KernelManager for customized behavior. """ ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Cache all the currently used ports self.currently_used_ports = set() @observe('kernel_manager_class') def _kernel_manager_class_changed(self, change): self.kernel_manager_factory = self._create_kernel_manager_factory() kernel_manager_factory = Any(help="this is kernel_manager_class after import") @default('kernel_manager_factory') def _kernel_manager_factory_default(self): return self._create_kernel_manager_factory() def _create_kernel_manager_factory(self): kernel_manager_ctor = import_item(self.kernel_manager_class) def create_kernel_manager(*args, **kwargs): if self.shared_context: if self.context.closed: # recreate context if closed self.context = self._context_default() kwargs.setdefault("context", self.context) km = kernel_manager_ctor(*args, **kwargs) if km.cache_ports: km.shell_port = self._find_available_port(km.ip) km.iopub_port = self._find_available_port(km.ip) km.stdin_port = self._find_available_port(km.ip) km.hb_port = self._find_available_port(km.ip) km.control_port = self._find_available_port(km.ip) return km return create_kernel_manager def _find_available_port(self, ip): while True: tmp_sock = socket.socket() tmp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8) tmp_sock.bind((ip, 0)) port = tmp_sock.getsockname()[1] tmp_sock.close() # This is a workaround for https://github.com/jupyter/jupyter_client/issues/487 # We prevent two kernels to have the same ports. if port not in self.currently_used_ports: self.currently_used_ports.add(port) return port shared_context = Bool( True, config=True, help="Share a single zmq.Context to talk to all my kernels", ) _created_context = Bool(False) context = Instance('zmq.Context') @default("context") def _context_default(self): self._created_context = True return zmq.Context() def __del__(self): if self._created_context and self.context and not self.context.closed: if self.log: self.log.debug("Destroying zmq context for %s", self) self.context.destroy() try: super_del = super().__del__ except AttributeError: pass else: super_del() connection_dir = Unicode('') _kernels = Dict() def list_kernel_ids(self): """Return a list of the kernel ids of the active kernels.""" # Create a copy so we can iterate over kernels in operations # that delete keys. return list(self._kernels.keys()) def __len__(self): """Return the number of running kernels.""" return len(self.list_kernel_ids()) def __contains__(self, kernel_id): return kernel_id in self._kernels def pre_start_kernel(self, kernel_name, kwargs): # kwargs should be mutable, passing it as a dict argument. kernel_id = kwargs.pop('kernel_id', self.new_kernel_id(**kwargs)) if kernel_id in self: raise DuplicateKernelError('Kernel already exists: %s' % kernel_id) if kernel_name is None: kernel_name = self.default_kernel_name # kernel_manager_factory is the constructor for the KernelManager # subclass we are using. It can be configured as any Configurable, # including things like its transport and ip. constructor_kwargs = {} if self.kernel_spec_manager: constructor_kwargs['kernel_spec_manager'] = self.kernel_spec_manager km = self.kernel_manager_factory(connection_file=os.path.join( self.connection_dir, "kernel-%s.json" % kernel_id), parent=self, log=self.log, kernel_name=kernel_name, **constructor_kwargs ) return km, kernel_name, kernel_id def start_kernel(self, kernel_name=None, **kwargs): """Start a new kernel. The caller can pick a kernel_id by passing one in as a keyword arg, otherwise one will be generated using new_kernel_id(). The kernel ID for the newly started kernel is returned. """ km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) km.start_kernel(**kwargs) self._kernels[kernel_id] = km return kernel_id def shutdown_kernel(self, kernel_id, now=False, restart=False): """Shutdown a kernel by its kernel uuid. Parameters ========== kernel_id : uuid The id of the kernel to shutdown. now : bool Should the kernel be shutdown forcibly using a signal. restart : bool Will the kernel be restarted? """ self.log.info("Kernel shutdown: %s" % kernel_id) km = self.get_kernel(kernel_id) ports = ( km.shell_port, km.iopub_port, km.stdin_port, km.hb_port, km.control_port ) km.shutdown_kernel(now=now, restart=restart) self.remove_kernel(kernel_id) if km.cache_ports and not restart: for port in ports: self.currently_used_ports.remove(port) @kernel_method def request_shutdown(self, kernel_id, restart=False): """Ask a kernel to shut down by its kernel uuid""" @kernel_method def finish_shutdown(self, kernel_id, waittime=None, pollinterval=0.1): """Wait for a kernel to finish shutting down, and kill it if it doesn't """ self.log.info("Kernel shutdown: %s" % kernel_id) @kernel_method def cleanup(self, kernel_id, connection_file=True): """Clean up a kernel's resources""" @kernel_method def cleanup_resources(self, kernel_id, restart=False): """Clean up a kernel's resources""" def remove_kernel(self, kernel_id): """remove a kernel from our mapping. Mainly so that a kernel can be removed if it is already dead, without having to call shutdown_kernel. The kernel object is returned. """ return self._kernels.pop(kernel_id) def shutdown_all(self, now=False): """Shutdown all kernels.""" kids = self.list_kernel_ids() for kid in kids: self.request_shutdown(kid) for kid in kids: self.finish_shutdown(kid) # Determine which cleanup method to call # See comment in KernelManager.shutdown_kernel(). km = self.get_kernel(kid) overrides_cleanup = type(km).cleanup is not KernelManager.cleanup overrides_cleanup_resources = type(km).cleanup_resources is not KernelManager.cleanup_resources if overrides_cleanup and not overrides_cleanup_resources: km.cleanup(connection_file=True) else: km.cleanup_resources(restart=False) self.remove_kernel(kid) @kernel_method def interrupt_kernel(self, kernel_id): """Interrupt (SIGINT) the kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel interrupted: %s" % kernel_id) @kernel_method def signal_kernel(self, kernel_id, signum): """Sends a signal to the kernel by its uuid. Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. Parameters ========== kernel_id : uuid The id of the kernel to signal. """ self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum)) @kernel_method def restart_kernel(self, kernel_id, now=False): """Restart a kernel by its uuid, keeping the same ports. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel restarted: %s" % kernel_id) @kernel_method def is_alive(self, kernel_id): """Is the kernel alive. This calls KernelManager.is_alive() which calls Popen.poll on the actual kernel subprocess. Parameters ========== kernel_id : uuid The id of the kernel. """ def _check_kernel_id(self, kernel_id): """check that a kernel id is valid""" if kernel_id not in self: raise KeyError("Kernel with id not found: %s" % kernel_id) def get_kernel(self, kernel_id): """Get the single KernelManager object for a kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel. """ self._check_kernel_id(kernel_id) return self._kernels[kernel_id] @kernel_method def add_restart_callback(self, kernel_id, callback, event='restart'): """add a callback for the KernelRestarter""" @kernel_method def remove_restart_callback(self, kernel_id, callback, event='restart'): """remove a callback for the KernelRestarter""" @kernel_method def get_connection_info(self, kernel_id): """Return a dictionary of connection data for a kernel. Parameters ========== kernel_id : uuid The id of the kernel. Returns ======= connection_dict : dict A dict of the information needed to connect to a kernel. This includes the ip address and the integer port numbers of the different channels (stdin_port, iopub_port, shell_port, hb_port). """ @kernel_method def connect_iopub(self, kernel_id, identity=None): """Return a zmq Socket connected to the iopub channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_shell(self, kernel_id, identity=None): """Return a zmq Socket connected to the shell channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_control(self, kernel_id, identity=None): """Return a zmq Socket connected to the control channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_stdin(self, kernel_id, identity=None): """Return a zmq Socket connected to the stdin channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_hb(self, kernel_id, identity=None): """Return a zmq Socket connected to the hb channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ def new_kernel_id(self, **kwargs): """ Returns the id to associate with the kernel for this request. Subclasses may override this method to substitute other sources of kernel ids. :param kwargs: :return: string-ized version 4 uuid """ return str(uuid.uuid4())
class HTTPKernelManager(AsyncKernelManager): """Manages a single kernel remotely via a Gateway Server. """ kernel_id = None kernel = None def __init__(self, **kwargs): super().__init__(**kwargs) self.base_endpoint = url_path_join( GatewayClient.instance().url, GatewayClient.instance().kernels_endpoint) self.kernel = None def _get_kernel_endpoint_url(self, kernel_id=None): """Builds a url for the kernels endpoint Parameters ---------- kernel_id: kernel UUID (optional) """ if kernel_id: return url_path_join(self.base_endpoint, url_escape(str(kernel_id))) return self.base_endpoint @property def has_kernel(self): """Has a kernel been started that we are managing.""" return self.kernel is not None client_class = DottedObjectName( 'elyra.pipeline.http_kernel_manager.HTTPKernelClient') client_factory = Type( klass='elyra.pipeline.http_kernel_manager.HTTPKernelClient') # -------------------------------------------------------------------------- # create a Client connected to our Kernel # -------------------------------------------------------------------------- def client(self, **kwargs): """Create a client configured to connect to our kernel""" kw = {} kw.update(self.get_connection_info(session=True)) kw.update(dict( connection_file=self.connection_file, parent=self, )) kw['kernel_id'] = self.kernel_id # add kwargs last, for manual overrides kw.update(kwargs) return self.client_factory(**kw) async def get_kernel(self, kernel_id): """Get kernel for kernel_id. Parameters ---------- kernel_id : uuid The uuid of the kernel. """ kernel_url = self._get_kernel_endpoint_url(kernel_id) self.log.debug("Request kernel at: %s" % kernel_url) try: response = await gateway_request(kernel_url, method='GET') except web.HTTPError as error: if error.status_code == 404: self.log.warning("Kernel not found at: %s" % kernel_url) kernel = None else: raise else: kernel = json_decode(response.body) self.log.debug("Kernel retrieved: %s" % kernel) return kernel # -------------------------------------------------------------------------- # Kernel management # -------------------------------------------------------------------------- async def start_kernel(self, **kwargs): """Starts a kernel via HTTP in an asynchronous manner. Parameters ---------- `**kwargs` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ kernel_id = kwargs.get('kernel_id') if kernel_id is None: kernel_name = kwargs.get('kernel_name', 'python3') kernel_url = self._get_kernel_endpoint_url() self.log.debug("Request new kernel at: %s" % kernel_url) # Let KERNEL_USERNAME take precedent over http_user config option. if os.environ.get('KERNEL_USERNAME' ) is None and GatewayClient.instance().http_user: os.environ['KERNEL_USERNAME'] = GatewayClient.instance( ).http_user kernel_env = { k: v for (k, v) in dict(os.environ).items() if k.startswith('KERNEL_') or k in GatewayClient.instance().env_whitelist.split(",") } # Add any env entries in this request kernel_env.update(kwargs.get('env')) # Convey the full path to where this notebook file is located. if kwargs.get('cwd') is not None and kernel_env.get( 'KERNEL_WORKING_DIR') is None: kernel_env['KERNEL_WORKING_DIR'] = kwargs['cwd'] json_body = json_encode({'name': kernel_name, 'env': kernel_env}) response = await gateway_request(kernel_url, method='POST', body=json_body) self.kernel = json_decode(response.body) self.kernel_id = self.kernel['id'] self.log.info( "HTTPKernelManager started kernel: {}, args: {}".format( self.kernel_id, kwargs)) else: self.kernel = await self.get_kernel(kernel_id) self.kernel_id = self.kernel['id'] self.log.info("HTTPKernelManager using existing kernel: {}".format( self.kernel_id)) async def shutdown_kernel(self, now=False, restart=False): """Attempts to stop the kernel process cleanly via HTTP. """ if self.has_kernel: kernel_url = self._get_kernel_endpoint_url(self.kernel_id) self.log.debug("Request shutdown kernel at: %s", kernel_url) response = await gateway_request(kernel_url, method='DELETE') self.log.debug("Shutdown kernel response: %d %s", response.code, response.reason) async def restart_kernel(self, **kw): """Restarts a kernel via HTTP. """ if self.has_kernel: kernel_url = self._get_kernel_endpoint_url( self.kernel_id) + '/restart' self.log.debug("Request restart kernel at: %s", kernel_url) response = await gateway_request(kernel_url, method='POST', body=json_encode({})) self.log.debug("Restart kernel response: %d %s", response.code, response.reason) async def interrupt_kernel(self): """Interrupts the kernel via an HTTP request. """ if self.has_kernel: kernel_url = self._get_kernel_endpoint_url( self.kernel_id) + '/interrupt' self.log.debug("Request interrupt kernel at: %s", kernel_url) response = await gateway_request(kernel_url, method='POST', body=json_encode({})) self.log.debug("Interrupt kernel response: %d %s", response.code, response.reason) async def is_alive(self): """Is the kernel process still running?""" if self.has_kernel: # Go ahead and issue a request to get the kernel self.kernel = await self.get_kernel(self.kernel_id) return True else: # we don't have a kernel return False def cleanup_resources(self, restart=False): """Clean up resources when the kernel is shut down""" pass
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp, ConnectionFileMixin): name='ipython-kernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session] # the kernel class, as an importstring kernel_class = Type('ipykernel.ipkernel.IPythonKernel', klass='ipykernel.kernelbase.Kernel', help="""The Kernel subclass to be used. This should allow easy re-use of the IPKernelApp entry point to configure and launch kernels other than IPython's own. """).tag(config=True) kernel = Any() poller = Any() # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat, allow_none=True) context = Any() shell_socket = Any() control_socket = Any() debugpy_socket = Any() debug_shell_socket = Any() stdin_socket = Any() iopub_socket = Any() iopub_thread = Any() control_thread = Any() _ports = Dict() subcommands = { 'install': ( 'ipykernel.kernelspec.InstallIPythonKernelSpecApp', 'Install the IPython kernel' ), } # connection info: connection_dir = Unicode() @default('connection_dir') def _default_connection_dir(self): return jupyter_runtime_dir() @property def abs_connection_file(self): if os.path.basename(self.connection_file) == self.connection_file: return os.path.join(self.connection_dir, self.connection_file) else: return self.connection_file # streams, etc. no_stdout = Bool(False, help="redirect stdout to the null device").tag(config=True) no_stderr = Bool(False, help="redirect stderr to the null device").tag(config=True) trio_loop = Bool(False, help="Set main event loop.").tag(config=True) quiet = Bool(True, help="Only send stdout/stderr to output stream").tag(config=True) outstream_class = DottedObjectName('ipykernel.iostream.OutStream', help="The importstring for the OutStream factory").tag(config=True) displayhook_class = DottedObjectName('ipykernel.displayhook.ZMQDisplayHook', help="The importstring for the DisplayHook factory").tag(config=True) capture_fd_output = Bool( True, help="""Attempt to capture and forward low-level output, e.g. produced by Extension libraries. """, ).tag(config=True) # polling parent_handle = Integer(int(os.environ.get('JPY_PARENT_PID') or 0), help="""kill this process if its parent dies. On Windows, the argument specifies the HANDLE of the parent process, otherwise it is simply boolean. """).tag(config=True) interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0), help="""ONLY USED ON WINDOWS Interrupt this process when the parent is signaled. """).tag(config=True) def init_crash_handler(self): sys.excepthook = self.excepthook def excepthook(self, etype, evalue, tb): # write uncaught traceback to 'real' stderr, not zmq-forwarder traceback.print_exception(etype, evalue, tb, file=sys.__stderr__) def init_poller(self): if sys.platform == 'win32': if self.interrupt or self.parent_handle: self.poller = ParentPollerWindows(self.interrupt, self.parent_handle) elif self.parent_handle and self.parent_handle != 1: # PID 1 (init) is special and will never go away, # only be reassigned. # Parent polling doesn't work if ppid == 1 to start with. self.poller = ParentPollerUnix() def _try_bind_socket(self, s, port): iface = '%s://%s' % (self.transport, self.ip) if self.transport == 'tcp': if port <= 0: port = s.bind_to_random_port(iface) else: s.bind("tcp://%s:%i" % (self.ip, port)) elif self.transport == 'ipc': if port <= 0: port = 1 path = "%s-%i" % (self.ip, port) while os.path.exists(path): port = port + 1 path = "%s-%i" % (self.ip, port) else: path = "%s-%i" % (self.ip, port) s.bind("ipc://%s" % path) return port def _bind_socket(self, s, port): try: win_in_use = errno.WSAEADDRINUSE except AttributeError: win_in_use = None # Try up to 100 times to bind a port when in conflict to avoid # infinite attempts in bad setups max_attempts = 1 if port else 100 for attempt in range(max_attempts): try: return self._try_bind_socket(s, port) except zmq.ZMQError as ze: # Raise if we have any error not related to socket binding if ze.errno != errno.EADDRINUSE and ze.errno != win_in_use: raise if attempt == max_attempts - 1: raise def write_connection_file(self): """write connection info to JSON file""" cf = self.abs_connection_file self.log.debug("Writing connection file: %s", cf) write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port, control_port=self.control_port) def cleanup_connection_file(self): cf = self.abs_connection_file self.log.debug("Cleaning up connection file: %s", cf) try: os.remove(cf) except (IOError, OSError): pass self.cleanup_ipc_files() def init_connection_file(self): if not self.connection_file: self.connection_file = "kernel-%s.json"%os.getpid() try: self.connection_file = filefind(self.connection_file, ['.', self.connection_dir]) except IOError: self.log.debug("Connection file not found: %s", self.connection_file) # This means I own it, and I'll create it in this directory: os.makedirs(os.path.dirname(self.abs_connection_file), mode=0o700, exist_ok=True) # Also, I will clean it up: atexit.register(self.cleanup_connection_file) return try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_sockets(self): # Create a context, a session, and the kernel sockets. self.log.info("Starting the kernel at pid: %i", os.getpid()) assert self.context is None, "init_sockets cannot be called twice!" self.context = context = zmq.Context() atexit.register(self.close) self.shell_socket = context.socket(zmq.ROUTER) self.shell_socket.linger = 1000 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_socket.linger = 1000 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) if hasattr(zmq, 'ROUTER_HANDOVER'): # set router-handover to workaround zeromq reconnect problems # in certain rare circumstances # see ipython/ipykernel#270 and zeromq/libzmq#2892 self.shell_socket.router_handover = \ self.stdin_socket.router_handover = 1 self.init_control(context) self.init_iopub(context) def init_control(self, context): self.control_socket = context.socket(zmq.ROUTER) self.control_socket.linger = 1000 self.control_port = self._bind_socket(self.control_socket, self.control_port) self.log.debug("control ROUTER Channel on port: %i" % self.control_port) self.debugpy_socket = context.socket(zmq.STREAM) self.debugpy_socket.linger = 1000 self.debug_shell_socket = context.socket(zmq.DEALER) self.debug_shell_socket.linger = 1000 if self.shell_socket.getsockopt(zmq.LAST_ENDPOINT): self.debug_shell_socket.connect(self.shell_socket.getsockopt(zmq.LAST_ENDPOINT)) if hasattr(zmq, 'ROUTER_HANDOVER'): # set router-handover to workaround zeromq reconnect problems # in certain rare circumstances # see ipython/ipykernel#270 and zeromq/libzmq#2892 self.control_socket.router_handover = 1 self.control_thread = ControlThread(daemon=True) def init_iopub(self, context): self.iopub_socket = context.socket(zmq.PUB) self.iopub_socket.linger = 1000 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) self.configure_tornado_logger() self.iopub_thread = IOPubThread(self.iopub_socket, pipe=True) self.iopub_thread.start() # backward-compat: wrap iopub socket API in background thread self.iopub_socket = self.iopub_thread.background_socket def init_heartbeat(self): """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) self.heartbeat.start() def close(self): """Close zmq sockets in an orderly fashion""" # un-capture IO before we start closing channels self.reset_io() self.log.info("Cleaning up sockets") if self.heartbeat: self.log.debug("Closing heartbeat channel") self.heartbeat.context.term() if self.iopub_thread: self.log.debug("Closing iopub channel") self.iopub_thread.stop() self.iopub_thread.close() if self.control_thread and self.control_thread.is_alive(): self.log.debug("Closing control thread") self.control_thread.stop() self.control_thread.join() if self.debugpy_socket and not self.debugpy_socket.closed: self.debugpy_socket.close() if self.debug_shell_socket and not self.debug_shell_socket.closed: self.debug_shell_socket.close() for channel in ('shell', 'control', 'stdin'): self.log.debug("Closing %s channel", channel) socket = getattr(self, channel + "_socket", None) if socket and not socket.closed: socket.close() self.log.debug("Terminating zmq context") self.context.term() self.log.debug("Terminated zmq context") def log_connection_info(self): """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.connection_dir: # use shortname tail = basename else: tail = self.connection_file lines = [ "To connect another client to this kernel, use:", " --existing %s" % tail, ] # log connection info # info-level, so often not shown. # frontends should use the %connect_info magic # to see the connection info for line in lines: self.log.info(line) # also raw print to the terminal if no parent_handle (`ipython kernel`) # unless log-level is CRITICAL (--quiet) if not self.parent_handle and self.log_level < logging.CRITICAL: print(_ctrl_c_message, file=sys.__stdout__) for line in lines: print(line, file=sys.__stdout__) self._ports = dict(shell=self.shell_port, iopub=self.iopub_port, stdin=self.stdin_port, hb=self.hb_port, control=self.control_port) def init_blackhole(self): """redirects stdout/stderr to devnull if necessary""" if self.no_stdout or self.no_stderr: blackhole = open(os.devnull, 'w') if self.no_stdout: sys.stdout = sys.__stdout__ = blackhole if self.no_stderr: sys.stderr = sys.__stderr__ = blackhole def init_io(self): """Redirect input streams and set a display hook.""" if self.outstream_class: outstream_factory = import_item(str(self.outstream_class)) if sys.stdout is not None: sys.stdout.flush() e_stdout = None if self.quiet else sys.__stdout__ e_stderr = None if self.quiet else sys.__stderr__ if not self.capture_fd_output: outstream_factory = partial(outstream_factory, watchfd=False) sys.stdout = outstream_factory(self.session, self.iopub_thread, 'stdout', echo=e_stdout) if sys.stderr is not None: sys.stderr.flush() sys.stderr = outstream_factory( self.session, self.iopub_thread, "stderr", echo=e_stderr ) if hasattr(sys.stderr, "_original_stdstream_copy"): for handler in self.log.handlers: if isinstance(handler, StreamHandler) and ( handler.stream.buffer.fileno() == 2 ): self.log.debug( "Seeing logger to stderr, rerouting to raw filedescriptor." ) handler.stream = TextIOWrapper( FileIO(sys.stderr._original_stdstream_copy, "w") ) if self.displayhook_class: displayhook_factory = import_item(str(self.displayhook_class)) self.displayhook = displayhook_factory(self.session, self.iopub_socket) sys.displayhook = self.displayhook self.patch_io() def reset_io(self): """restore original io restores state after init_io """ sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ sys.displayhook = sys.__displayhook__ def patch_io(self): """Patch important libraries that can't handle sys.stdout forwarding""" try: import faulthandler except ImportError: pass else: # Warning: this is a monkeypatch of `faulthandler.enable`, watch for possible # updates to the upstream API and update accordingly (up-to-date as of Python 3.5): # https://docs.python.org/3/library/faulthandler.html#faulthandler.enable # change default file to __stderr__ from forwarded stderr faulthandler_enable = faulthandler.enable def enable(file=sys.__stderr__, all_threads=True, **kwargs): return faulthandler_enable(file=file, all_threads=all_threads, **kwargs) faulthandler.enable = enable if hasattr(faulthandler, 'register'): faulthandler_register = faulthandler.register def register(signum, file=sys.__stderr__, all_threads=True, chain=False, **kwargs): return faulthandler_register(signum, file=file, all_threads=all_threads, chain=chain, **kwargs) faulthandler.register = register def init_signal(self): signal.signal(signal.SIGINT, signal.SIG_IGN) def init_kernel(self): """Create the Kernel object itself""" shell_stream = ZMQStream(self.shell_socket) control_stream = ZMQStream(self.control_socket, self.control_thread.io_loop) debugpy_stream = ZMQStream(self.debugpy_socket, self.control_thread.io_loop) self.control_thread.start() kernel_factory = self.kernel_class.instance kernel = kernel_factory(parent=self, session=self.session, control_stream=control_stream, debugpy_stream=debugpy_stream, debug_shell_socket=self.debug_shell_socket, shell_stream=shell_stream, control_thread=self.control_thread, iopub_thread=self.iopub_thread, iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, user_ns=self.user_ns, ) kernel.record_ports({ name + '_port': port for name, port in self._ports.items() }) self.kernel = kernel # Allow the displayhook to get the execution count self.displayhook.get_execution_count = lambda: kernel.execution_count def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" # Register inline backend as default # this is higher priority than matplotlibrc, # but lower priority than anything else (mpl.use() for instance). # This only affects matplotlib >= 1.5 if not os.environ.get('MPLBACKEND'): os.environ['MPLBACKEND'] = 'module://matplotlib_inline.backend_inline' # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` # to ensure that any exception is printed straight to stderr. # Normally _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. shell = self.shell _showtraceback = shell._showtraceback try: # replace error-sending traceback with stderr def print_tb(etype, evalue, stb): print ("GUI event loop or pylab initialization failed", file=sys.stderr) print (shell.InteractiveTB.stb2text(stb), file=sys.stderr) shell._showtraceback = print_tb InteractiveShellApp.init_gui_pylab(self) finally: shell._showtraceback = _showtraceback def init_shell(self): self.shell = getattr(self.kernel, 'shell', None) if self.shell: self.shell.configurables.append(self) def configure_tornado_logger(self): """ Configure the tornado logging.Logger. Must set up the tornado logger or else tornado will call basicConfig for the root logger which makes the root logger go to the real sys.stderr instead of the capture streams. This function mimics the setup of logging.basicConfig. """ logger = logging.getLogger('tornado') handler = logging.StreamHandler() formatter = logging.Formatter(logging.BASIC_FORMAT) handler.setFormatter(formatter) logger.addHandler(handler) def _init_asyncio_patch(self): """set default asyncio policy to be compatible with tornado Tornado 6 (at least) is not compatible with the default asyncio implementation on Windows Pick the older SelectorEventLoopPolicy on Windows if the known-incompatible default policy is in use. Support for Proactor via a background thread is available in tornado 6.1, but it is still preferable to run the Selector in the main thread instead of the background. do this as early as possible to make it a low priority and overrideable ref: https://github.com/tornadoweb/tornado/issues/2608 FIXME: if/when tornado supports the defaults in asyncio without threads, remove and bump tornado requirement for py38. Most likely, this will mean a new Python version where asyncio.ProactorEventLoop supports add_reader and friends. """ if sys.platform.startswith("win") and sys.version_info >= (3, 8): import asyncio try: from asyncio import ( WindowsProactorEventLoopPolicy, WindowsSelectorEventLoopPolicy, ) except ImportError: pass # not affected else: if type(asyncio.get_event_loop_policy()) is WindowsProactorEventLoopPolicy: # WindowsProactorEventLoopPolicy is not compatible with tornado 6 # fallback to the pre-3.8 default of Selector asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) def init_pdb(self): """Replace pdb with IPython's version that is interruptible. With the non-interruptible version, stopping pdb() locks up the kernel in a non-recoverable state. """ import pdb from IPython.core import debugger if hasattr(debugger, "InterruptiblePdb"): # Only available in newer IPython releases: debugger.Pdb = debugger.InterruptiblePdb pdb.Pdb = debugger.Pdb pdb.set_trace = debugger.set_trace @catch_config_error def initialize(self, argv=None): self._init_asyncio_patch() super(IPKernelApp, self).initialize(argv) if self.subapp is not None: return self.init_pdb() self.init_blackhole() self.init_connection_file() self.init_poller() self.init_sockets() self.init_heartbeat() # writing/displaying connection info must be *after* init_sockets/heartbeat self.write_connection_file() # Log connection info after writing connection file, so that the connection # file is definitely available at the time someone reads the log. self.log_connection_info() self.init_io() try: self.init_signal() except Exception: # Catch exception when initializing signal fails, eg when running the # kernel on a separate thread if self.log_level < logging.CRITICAL: self.log.error("Unable to initialize signal:", exc_info=True) self.init_kernel() # shell init steps self.init_path() self.init_shell() if self.shell: self.init_gui_pylab() self.init_extensions() self.init_code() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request sys.stdout.flush() sys.stderr.flush() def start(self): if self.subapp is not None: return self.subapp.start() if self.poller is not None: self.poller.start() self.kernel.start() self.io_loop = ioloop.IOLoop.current() if self.trio_loop: from ipykernel.trio_runner import TrioRunner tr = TrioRunner() tr.initialize(self.kernel, self.io_loop) try: tr.run() except KeyboardInterrupt: pass else: try: self.io_loop.start() except KeyboardInterrupt: pass
class KernelManager(ConnectionFileMixin): """Manages a single kernel in a subprocess on this host. This version starts kernels with Popen. """ def __init__(self, *args, **kwargs): super().__init__(**kwargs) self._shutdown_status = _ShutdownStatus.Unset _created_context: Bool = Bool(False) # The PyZMQ Context to use for communication with the kernel. context: Instance = Instance(zmq.Context) @default("context") def _context_default(self) -> zmq.Context: self._created_context = True return zmq.Context() # the class to create with our `client` method client_class: DottedObjectName = DottedObjectName( "jupyter_client.blocking.BlockingKernelClient" ) client_factory: Type = Type(klass="jupyter_client.KernelClient") @default("client_factory") def _client_factory_default(self) -> Type: return import_item(self.client_class) @observe("client_class") def _client_class_changed(self, change: t.Dict[str, DottedObjectName]) -> None: self.client_factory = import_item(str(change["new"])) kernel_id: str = Unicode(None, allow_none=True) # The kernel provisioner with which this KernelManager is communicating. # This will generally be a LocalProvisioner instance unless the kernelspec # indicates otherwise. provisioner: t.Optional[KernelProvisionerBase] = None kernel_spec_manager: Instance = Instance(kernelspec.KernelSpecManager) @default("kernel_spec_manager") def _kernel_spec_manager_default(self) -> kernelspec.KernelSpecManager: return kernelspec.KernelSpecManager(data_dir=self.data_dir) @observe("kernel_spec_manager") @observe_compat def _kernel_spec_manager_changed(self, change: t.Dict[str, Instance]) -> None: self._kernel_spec = None shutdown_wait_time: Float = Float( 5.0, config=True, help="Time to wait for a kernel to terminate before killing it, " "in seconds. When a shutdown request is initiated, the kernel " "will be immediately sent an interrupt (SIGINT), followed" "by a shutdown_request message, after 1/2 of `shutdown_wait_time`" "it will be sent a terminate (SIGTERM) request, and finally at " "the end of `shutdown_wait_time` will be killed (SIGKILL). terminate " "and kill may be equivalent on windows. Note that this value can be" "overridden by the in-use kernel provisioner since shutdown times may" "vary by provisioned environment.", ) kernel_name: Unicode = Unicode(kernelspec.NATIVE_KERNEL_NAME) @observe("kernel_name") def _kernel_name_changed(self, change: t.Dict[str, Unicode]) -> None: self._kernel_spec = None if change["new"] == "python": self.kernel_name = kernelspec.NATIVE_KERNEL_NAME _kernel_spec: t.Optional[kernelspec.KernelSpec] = None @property def kernel_spec(self) -> t.Optional[kernelspec.KernelSpec]: if self._kernel_spec is None and self.kernel_name != "": self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec cache_ports: Bool = Bool( help="True if the MultiKernelManager should cache ports for this KernelManager instance" ) @default("cache_ports") def _default_cache_ports(self) -> bool: return self.transport == "tcp" @property def ipykernel(self) -> bool: return self.kernel_name in {"python", "python2", "python3"} # Protected traits _launch_args: Any = Any() _control_socket: Any = Any() _restarter: Any = Any() autorestart: Bool = Bool( True, config=True, help="""Should we autorestart the kernel if it dies.""" ) shutting_down: bool = False def __del__(self) -> None: self._close_control_socket() self.cleanup_connection_file() # -------------------------------------------------------------------------- # Kernel restarter # -------------------------------------------------------------------------- def start_restarter(self) -> None: pass def stop_restarter(self) -> None: pass def add_restart_callback(self, callback: t.Callable, event: str = "restart") -> None: """register a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.add_callback(callback, event) def remove_restart_callback(self, callback: t.Callable, event: str = "restart") -> None: """unregister a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.remove_callback(callback, event) # -------------------------------------------------------------------------- # create a Client connected to our Kernel # -------------------------------------------------------------------------- def client(self, **kwargs) -> KernelClient: """Create a client configured to connect to our kernel""" kw = {} kw.update(self.get_connection_info(session=True)) kw.update( dict( connection_file=self.connection_file, parent=self, ) ) # add kwargs last, for manual overrides kw.update(kwargs) return self.client_factory(**kw) # -------------------------------------------------------------------------- # Kernel management # -------------------------------------------------------------------------- def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> t.List[str]: """replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] assert self.kernel_spec is not None cmd = self.kernel_spec.argv + extra_arguments if cmd and cmd[0] in { "python", "python%i" % sys.version_info[0], "python%i.%i" % sys.version_info[:2], }: # executable is 'python' or 'python3', use sys.executable. # These will typically be the same, # but if the current process is in an env # and has been launched by abspath without # activating the env, python on PATH may not be sys.executable, # but it should be. cmd[0] = sys.executable # Make sure to use the realpath for the connection_file # On windows, when running with the store python, the connection_file path # is not usable by non python kernels because the path is being rerouted when # inside of a store app. # See this bug here: https://bugs.python.org/issue41196 ns = dict( connection_file=os.path.realpath(self.connection_file), prefix=sys.prefix, ) if self.kernel_spec: ns["resource_dir"] = self.kernel_spec.resource_dir ns.update(self._launch_args) pat = re.compile(r"\{([A-Za-z0-9_]+)\}") def from_ns(match): """Get the key out of ns if it's there, otherwise no change.""" return ns.get(match.group(1), match.group()) return [pat.sub(from_ns, arg) for arg in cmd] async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw) -> None: """actually launch the kernel override in a subclass to launch kernel subprocesses differently Note that provisioners can now be used to customize kernel environments and """ assert self.provisioner is not None connection_info = await self.provisioner.launch_kernel(kernel_cmd, **kw) assert self.provisioner.has_process # Provisioner provides the connection information. Load into kernel manager and write file. self._force_connection_info(connection_info) _launch_kernel = run_sync(_async_launch_kernel) # Control socket used for polite kernel shutdown def _connect_control_socket(self) -> None: if self._control_socket is None: self._control_socket = self._create_connected_socket("control") self._control_socket.linger = 100 def _close_control_socket(self) -> None: if self._control_socket is None: return self._control_socket.close() self._control_socket = None async def _async_pre_start_kernel(self, **kw) -> t.Tuple[t.List[str], t.Dict[str, t.Any]]: """Prepares a kernel for startup in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ self.shutting_down = False self.kernel_id = self.kernel_id or kw.pop('kernel_id', str(uuid.uuid4())) # save kwargs for use in restart self._launch_args = kw.copy() if self.provisioner is None: # will not be None on restarts self.provisioner = KPF.instance(parent=self.parent).create_provisioner_instance( self.kernel_id, self.kernel_spec, parent=self, ) kw = await self.provisioner.pre_launch(**kw) kernel_cmd = kw.pop('cmd') return kernel_cmd, kw pre_start_kernel = run_sync(_async_pre_start_kernel) async def _async_post_start_kernel(self, **kw) -> None: """Performs any post startup tasks relative to the kernel. Parameters ---------- `**kw` : optional keyword arguments that were used in the kernel process's launch. """ self.start_restarter() self._connect_control_socket() assert self.provisioner is not None await self.provisioner.post_launch(**kw) post_start_kernel = run_sync(_async_post_start_kernel) async def _async_start_kernel(self, **kw): """Starts a kernel on this host in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ kernel_cmd, kw = await ensure_async(self.pre_start_kernel(**kw)) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) await ensure_async(self._launch_kernel(kernel_cmd, **kw)) await ensure_async(self.post_start_kernel(**kw)) start_kernel = run_sync(_async_start_kernel) async def _async_request_shutdown(self, restart: bool = False) -> None: """Send a shutdown request via control channel""" content = dict(restart=restart) msg = self.session.msg("shutdown_request", content=content) # ensure control socket is connected self._connect_control_socket() self.session.send(self._control_socket, msg) assert self.provisioner is not None await self.provisioner.shutdown_requested(restart=restart) self._shutdown_status = _ShutdownStatus.ShutdownRequest request_shutdown = run_sync(_async_request_shutdown) async def _async_finish_shutdown( self, waittime: t.Optional[float] = None, pollinterval: float = 0.1, restart: t.Optional[bool] = False, ) -> None: """Wait for kernel shutdown, then kill process if it doesn't shutdown. This does not send shutdown requests - use :meth:`request_shutdown` first. """ if waittime is None: waittime = max(self.shutdown_wait_time, 0) if self.provisioner: # Allow provisioner to override waittime = self.provisioner.get_shutdown_wait_time(recommended=waittime) try: await asyncio.wait_for( self._async_wait(pollinterval=pollinterval), timeout=waittime / 2 ) except asyncio.TimeoutError: self.log.debug("Kernel is taking too long to finish, terminating") self._shutdown_status = _ShutdownStatus.SigtermRequest await ensure_async(self._send_kernel_sigterm()) # iOS: we cannot send a kill signal, so we just wait: if (sys.platform == "darwin" and os.uname().machine.startswith("iP")): return try: await asyncio.wait_for( self._async_wait(pollinterval=pollinterval), timeout=waittime / 2 ) except asyncio.TimeoutError: self.log.debug("Kernel is taking too long to finish, killing") self._shutdown_status = _ShutdownStatus.SigkillRequest await ensure_async(self._kill_kernel(restart=restart)) else: # Process is no longer alive, wait and clear if self.has_kernel: assert self.provisioner is not None await self.provisioner.wait() finish_shutdown = run_sync(_async_finish_shutdown) async def _async_cleanup_resources(self, restart: bool = False) -> None: """Clean up resources when the kernel is shut down""" if not restart: self.cleanup_connection_file() self.cleanup_ipc_files() self._close_control_socket() self.session.parent = None if self._created_context and not restart: self.context.destroy(linger=100) if self.provisioner: await self.provisioner.cleanup(restart=restart) cleanup_resources = run_sync(_async_cleanup_resources) async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False): """Attempts to stop the kernel process cleanly. This attempts to shutdown the kernels cleanly by: 1. Sending it a shutdown message over the control channel. 2. If that fails, the kernel is shutdown forcibly by sending it a signal. Parameters ---------- now : bool Should the kernel be forcible killed *now*. This skips the first, nice shutdown attempt. restart: bool Will this kernel be restarted after it is shutdown. When this is True, connection files will not be cleaned up. """ self.shutting_down = True # Used by restarter to prevent race condition # Stop monitoring for restarting while we shutdown. self.stop_restarter() await ensure_async(self.interrupt_kernel()) if now: await ensure_async(self._kill_kernel()) else: await ensure_async(self.request_shutdown(restart=restart)) # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. await ensure_async(self.finish_shutdown(restart=restart)) await ensure_async(self.cleanup_resources(restart=restart)) shutdown_kernel = run_sync(_async_shutdown_kernel) async def _async_restart_kernel(self, now: bool = False, newports: bool = False, **kw) -> None: """Restarts a kernel with the arguments that were used to launch it. Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. newports : bool, optional If the old kernel was launched with random ports, this flag decides whether the same ports and connection file will be used again. If False, the same ports and connection file are used. This is the default. If True, new random port numbers are chosen and a new connection file is written. It is still possible that the newly chosen random port numbers happen to be the same as the old ones. `**kw` : optional Any options specified here will overwrite those used to launch the kernel. """ if self._launch_args is None: raise RuntimeError("Cannot restart the kernel. " "No previous call to 'start_kernel'.") else: # Stop currently running kernel. await ensure_async(self.shutdown_kernel(now=now, restart=True)) if newports: self.cleanup_random_ports() # Start new kernel. self._launch_args.update(kw) await ensure_async(self.start_kernel(**self._launch_args)) restart_kernel = run_sync(_async_restart_kernel) @property def has_kernel(self) -> bool: """Has a kernel process been started that we are actively managing.""" return self.provisioner is not None and self.provisioner.has_process async def _async_send_kernel_sigterm(self, restart: bool = False) -> None: """similar to _kill_kernel, but with sigterm (not sigkill), but do not block""" if self.has_kernel: assert self.provisioner is not None await self.provisioner.terminate(restart=restart) _send_kernel_sigterm = run_sync(_async_send_kernel_sigterm) async def _async_kill_kernel(self, restart: bool = False) -> None: """Kill the running kernel. This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: assert self.provisioner is not None await self.provisioner.kill(restart=restart) # Wait until the kernel terminates. try: await asyncio.wait_for(self._async_wait(), timeout=5.0) except asyncio.TimeoutError: # Wait timed out, just log warning but continue - not much more we can do. self.log.warning("Wait for final termination of kernel timed out - continuing...") pass else: # Process is no longer alive, wait and clear if self.has_kernel: await self.provisioner.wait() _kill_kernel = run_sync(_async_kill_kernel) async def _async_interrupt_kernel(self) -> None: """Interrupts the kernel by sending it a signal. Unlike ``signal_kernel``, this operation is well supported on all platforms. """ if self.has_kernel: assert self.kernel_spec is not None interrupt_mode = self.kernel_spec.interrupt_mode if interrupt_mode == "signal": await ensure_async(self.signal_kernel(signal.SIGINT)) elif interrupt_mode == "message": msg = self.session.msg("interrupt_request", content={}) self._connect_control_socket() self.session.send(self._control_socket, msg) else: raise RuntimeError("Cannot interrupt kernel. No kernel is running!") interrupt_kernel = run_sync(_async_interrupt_kernel) async def _async_signal_kernel(self, signum: int) -> None: """Sends a signal to the process group of the kernel (this usually includes the kernel and any subprocesses spawned by the kernel). Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ if self.has_kernel: assert self.provisioner is not None await self.provisioner.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") signal_kernel = run_sync(_async_signal_kernel) async def _async_is_alive(self) -> bool: """Is the kernel process still running?""" if self.has_kernel: assert self.provisioner is not None ret = await self.provisioner.poll() if ret is None: return True return False is_alive = run_sync(_async_is_alive) async def _async_wait(self, pollinterval: float = 0.1) -> None: # Use busy loop at 100ms intervals, polling until the process is # not alive. If we find the process is no longer alive, complete # its cleanup via the blocking wait(). Callers are responsible for # issuing calls to wait() using a timeout (see _kill_kernel()). while await ensure_async(self.is_alive()): await asyncio.sleep(pollinterval)
class MultiQtKernelManager(MultiKernelManager): kernel_manager_class = DottedObjectName(__module__ + ".FixQtInProcessKernelManager", config = True, help = """kernel manager class""")
class Session(Configurable): """Object for handling serialization and sending of messages. The Session object handles building messages and sending them with ZMQ sockets or ZMQStream objects. Objects can communicate with each other over the network via Session objects, and only need to work with the dict-based IPython message spec. The Session will handle serialization/deserialization, security, and metadata. Sessions support configurable serialization via packer/unpacker traits, and signing with HMAC digests via the key/keyfile traits. Parameters ---------- debug : bool whether to trigger extra debugging statements packer/unpacker : str : 'json', 'pickle' or import_string importstrings for methods to serialize message parts. If just 'json' or 'pickle', predefined JSON and pickle packers will be used. Otherwise, the entire importstring must be used. The functions must accept at least valid JSON input, and output *bytes*. For example, to use msgpack: packer = 'msgpack.packb', unpacker='msgpack.unpackb' pack/unpack : callables You can also set the pack/unpack callables for serialization directly. session : bytes the ID of this Session object. The default is to generate a new UUID. username : unicode username added to message headers. The default is to ask the OS. key : bytes The key used to initialize an HMAC signature. If unset, messages will not be signed or checked. keyfile : filepath The file containing a key. If this is set, `key` will be initialized to the contents of the file. """ debug = Bool(False, config=True, help="""Debug output in the Session""") check_pid = Bool(True, config=True, help="""Whether to check PID to protect against calls after fork. This check can be disabled if fork-safety is handled elsewhere. """) packer = DottedObjectName('json',config=True, help="""The name of the packer for serializing messages. Should be one of 'json', 'pickle', or an import name for a custom callable serializer.""") @observe('packer') def _packer_changed(self, change): new = change['new'] if new.lower() == 'json': self.pack = json_packer self.unpack = json_unpacker self.unpacker = new elif new.lower() == 'pickle': self.pack = pickle_packer self.unpack = pickle_unpacker self.unpacker = new else: self.pack = import_item(str(new)) unpacker = DottedObjectName('json', config=True, help="""The name of the unpacker for unserializing messages. Only used with custom functions for `packer`.""") @observe('unpacker') def _unpacker_changed(self, change): new = change['new'] if new.lower() == 'json': self.pack = json_packer self.unpack = json_unpacker self.packer = new elif new.lower() == 'pickle': self.pack = pickle_packer self.unpack = pickle_unpacker self.packer = new else: self.unpack = import_item(str(new)) session = CUnicode(u'', config=True, help="""The UUID identifying this session.""") def _session_default(self): u = new_id() self.bsession = u.encode('ascii') return u @observe('session') def _session_changed(self, change): self.bsession = self.session.encode('ascii') # bsession is the session as bytes bsession = CBytes(b'') username = Unicode(str_to_unicode(os.environ.get('USER', 'username')), help="""Username for the Session. Default is your system username.""", config=True) metadata = Dict({}, config=True, help="""Metadata dictionary, which serves as the default top-level metadata dict for each message.""") # if 0, no adapting to do. adapt_version = Integer(0) # message signature related traits: key = CBytes(config=True, help="""execution key, for signing messages.""") def _key_default(self): return new_id_bytes() @observe('key') def _key_changed(self, change): self._new_auth() signature_scheme = Unicode('hmac-sha256', config=True, help="""The digest scheme used to construct the message signatures. Must have the form 'hmac-HASH'.""") @observe('signature_scheme') def _signature_scheme_changed(self, change): new = change['new'] if not new.startswith('hmac-'): raise TraitError("signature_scheme must start with 'hmac-', got %r" % new) hash_name = new.split('-', 1)[1] try: self.digest_mod = getattr(hashlib, hash_name) except AttributeError: raise TraitError("hashlib has no such attribute: %s" % hash_name) self._new_auth() digest_mod = Any() def _digest_mod_default(self): return hashlib.sha256 auth = Instance(hmac.HMAC, allow_none=True) def _new_auth(self): if self.key: self.auth = hmac.HMAC(self.key, digestmod=self.digest_mod) else: self.auth = None digest_history = Set() digest_history_size = Integer(2**16, config=True, help="""The maximum number of digests to remember. The digest history will be culled when it exceeds this value. """ ) keyfile = Unicode('', config=True, help="""path to file containing execution key.""") @observe('keyfile') def _keyfile_changed(self, change): with open(change['new'], 'rb') as f: self.key = f.read().strip() # for protecting against sends from forks pid = Integer() # serialization traits: pack = Any(default_packer) # the actual packer function @observe('pack') def _pack_changed(self, change): new = change['new'] if not callable(new): raise TypeError("packer must be callable, not %s"%type(new)) unpack = Any(default_unpacker) # the actual packer function @observe('unpack') def _unpack_changed(self, change): # unpacker is not checked - it is assumed to be new = change['new'] if not callable(new): raise TypeError("unpacker must be callable, not %s"%type(new)) # thresholds: copy_threshold = Integer(2**16, config=True, help="Threshold (in bytes) beyond which a buffer should be sent without copying.") buffer_threshold = Integer(MAX_BYTES, config=True, help="Threshold (in bytes) beyond which an object's buffer should be extracted to avoid pickling.") item_threshold = Integer(MAX_ITEMS, config=True, help="""The maximum number of items for a container to be introspected for custom serialization. Containers larger than this are pickled outright. """ ) def __init__(self, **kwargs): """create a Session object Parameters ---------- debug : bool whether to trigger extra debugging statements packer/unpacker : str : 'json', 'pickle' or import_string importstrings for methods to serialize message parts. If just 'json' or 'pickle', predefined JSON and pickle packers will be used. Otherwise, the entire importstring must be used. The functions must accept at least valid JSON input, and output *bytes*. For example, to use msgpack: packer = 'msgpack.packb', unpacker='msgpack.unpackb' pack/unpack : callables You can also set the pack/unpack callables for serialization directly. session : unicode (must be ascii) the ID of this Session object. The default is to generate a new UUID. bsession : bytes The session as bytes username : unicode username added to message headers. The default is to ask the OS. key : bytes The key used to initialize an HMAC signature. If unset, messages will not be signed or checked. signature_scheme : str The message digest scheme. Currently must be of the form 'hmac-HASH', where 'HASH' is a hashing function available in Python's hashlib. The default is 'hmac-sha256'. This is ignored if 'key' is empty. keyfile : filepath The file containing a key. If this is set, `key` will be initialized to the contents of the file. """ super(Session, self).__init__(**kwargs) self._check_packers() self.none = self.pack({}) # ensure self._session_default() if necessary, so bsession is defined: self.session self.pid = os.getpid() self._new_auth() if not self.key: get_logger().warning("Message signing is disabled. This is insecure and not recommended!") def clone(self): """Create a copy of this Session Useful when connecting multiple times to a given kernel. This prevents a shared digest_history warning about duplicate digests due to multiple connections to IOPub in the same process. .. versionadded:: 5.1 """ # make a copy new_session = type(self)() for name in self.traits(): setattr(new_session, name, getattr(self, name)) # fork digest_history new_session.digest_history = set() new_session.digest_history.update(self.digest_history) return new_session @property def msg_id(self): """always return new uuid""" return new_id() def _check_packers(self): """check packers for datetime support.""" pack = self.pack unpack = self.unpack # check simple serialization msg = dict(a=[1,'hi']) try: packed = pack(msg) except Exception as e: msg = "packer '{packer}' could not serialize a simple message: {e}{jsonmsg}" if self.packer == 'json': jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod else: jsonmsg = "" raise ValueError( msg.format(packer=self.packer, e=e, jsonmsg=jsonmsg) ) # ensure packed message is bytes if not isinstance(packed, bytes): raise ValueError("message packed to %r, but bytes are required"%type(packed)) # check that unpack is pack's inverse try: unpacked = unpack(packed) assert unpacked == msg except Exception as e: msg = "unpacker '{unpacker}' could not handle output from packer '{packer}': {e}{jsonmsg}" if self.packer == 'json': jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod else: jsonmsg = "" raise ValueError( msg.format(packer=self.packer, unpacker=self.unpacker, e=e, jsonmsg=jsonmsg) ) # check datetime support msg = dict(t=utcnow()) try: unpacked = unpack(pack(msg)) if isinstance(unpacked['t'], datetime): raise ValueError("Shouldn't deserialize to datetime") except Exception: self.pack = lambda o: pack(squash_dates(o)) self.unpack = lambda s: unpack(s) def msg_header(self, msg_type): return msg_header(self.msg_id, msg_type, self.username, self.session) def msg(self, msg_type, content=None, parent=None, header=None, metadata=None): """Return the nested message dict. This format is different from what is sent over the wire. The serialize/deserialize methods converts this nested message dict to the wire format, which is a list of message parts. """ msg = {} header = self.msg_header(msg_type) if header is None else header msg['header'] = header msg['msg_id'] = header['msg_id'] msg['msg_type'] = header['msg_type'] msg['parent_header'] = {} if parent is None else extract_header(parent) msg['content'] = {} if content is None else content msg['metadata'] = self.metadata.copy() if metadata is not None: msg['metadata'].update(metadata) return msg def sign(self, msg_list): """Sign a message with HMAC digest. If no auth, return b''. Parameters ---------- msg_list : list The [p_header,p_parent,p_content] part of the message list. """ if self.auth is None: return b'' h = self.auth.copy() for m in msg_list: h.update(m) return str_to_bytes(h.hexdigest()) def serialize(self, msg, ident=None): """Serialize the message components to bytes. This is roughly the inverse of deserialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg : dict or Message The next message dict as returned by the self.msg method. Returns ------- msg_list : list The list of bytes objects to be sent with the format:: [ident1, ident2, ..., DELIM, HMAC, p_header, p_parent, p_metadata, p_content, buffer1, buffer2, ...] In this list, the ``p_*`` entities are the packed or serialized versions, so if JSON is used, these are utf8 encoded JSON strings. """ content = msg.get('content', {}) if content is None: content = self.none elif isinstance(content, dict): content = self.pack(content) elif isinstance(content, bytes): # content is already packed, as in a relayed message pass elif isinstance(content, unicode_type): # should be bytes, but JSON often spits out unicode content = content.encode('utf8') else: raise TypeError("Content incorrect type: %s"%type(content)) real_message = [self.pack(msg['header']), self.pack(msg['parent_header']), self.pack(msg['metadata']), content, ] to_send = [] if isinstance(ident, list): # accept list of idents to_send.extend(ident) elif ident is not None: to_send.append(ident) to_send.append(DELIM) signature = self.sign(real_message) to_send.append(signature) to_send.extend(real_message) return to_send def send(self, stream, msg_or_type, content=None, parent=None, ident=None, buffers=None, track=False, header=None, metadata=None): """Build and send a message via stream or socket. The message format used by this function internally is as follows: [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content, buffer1,buffer2,...] The serialize/deserialize methods convert the nested message dict into this format. Parameters ---------- stream : zmq.Socket or ZMQStream The socket-like object used to send the data. msg_or_type : str or Message/dict Normally, msg_or_type will be a msg_type unless a message is being sent more than once. If a header is supplied, this can be set to None and the msg_type will be pulled from the header. content : dict or None The content of the message (ignored if msg_or_type is a message). header : dict or None The header dict for the message (ignored if msg_to_type is a message). parent : Message or dict or None The parent or parent header describing the parent of this message (ignored if msg_or_type is a message). ident : bytes or list of bytes The zmq.IDENTITY routing path. metadata : dict or None The metadata describing the message buffers : list or None The already-serialized buffers to be appended to the message. track : bool Whether to track. Only for use with Sockets, because ZMQStream objects cannot track messages. Returns ------- msg : dict The constructed message. """ if not isinstance(stream, zmq.Socket): # ZMQStreams and dummy sockets do not support tracking. track = False if isinstance(msg_or_type, (Message, dict)): # We got a Message or message dict, not a msg_type so don't # build a new Message. msg = msg_or_type buffers = buffers or msg.get('buffers', []) else: msg = self.msg(msg_or_type, content=content, parent=parent, header=header, metadata=metadata) if self.check_pid and not os.getpid() == self.pid: get_logger().warning("WARNING: attempted to send message from fork\n%s", msg ) return buffers = [] if buffers is None else buffers for idx, buf in enumerate(buffers): if isinstance(buf, memoryview): view = buf else: try: # check to see if buf supports the buffer protocol. view = memoryview(buf) except TypeError: raise TypeError("Buffer objects must support the buffer protocol.") # memoryview.contiguous is new in 3.3, # just skip the check on Python 2 if hasattr(view, 'contiguous') and not view.contiguous: # zmq requires memoryviews to be contiguous raise ValueError("Buffer %i (%r) is not contiguous" % (idx, buf)) if self.adapt_version: msg = adapt(msg, self.adapt_version) to_send = self.serialize(msg, ident) to_send.extend(buffers) longest = max([ len(s) for s in to_send ]) copy = (longest < self.copy_threshold) if buffers and track and not copy: # only really track when we are doing zero-copy buffers tracker = stream.send_multipart(to_send, copy=False, track=True) else: # use dummy tracker, which will be done immediately tracker = DONE stream.send_multipart(to_send, copy=copy) if self.debug: pprint.pprint(msg) pprint.pprint(to_send) pprint.pprint(buffers) msg['tracker'] = tracker return msg def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None): """Send a raw message via ident path. This method is used to send a already serialized message. Parameters ---------- stream : ZMQStream or Socket The ZMQ stream or socket to use for sending the message. msg_list : list The serialized list of messages to send. This only includes the [p_header,p_parent,p_metadata,p_content,buffer1,buffer2,...] portion of the message. ident : ident or list A single ident or a list of idents to use in sending. """ to_send = [] if isinstance(ident, bytes): ident = [ident] if ident is not None: to_send.extend(ident) to_send.append(DELIM) # Don't include buffers in signature (per spec). to_send.append(self.sign(msg_list[0:4])) to_send.extend(msg_list) stream.send_multipart(to_send, flags, copy=copy) def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True): """Receive and unpack a message. Parameters ---------- socket : ZMQStream or Socket The socket or stream to use in receiving. Returns ------- [idents], msg [idents] is a list of idents and msg is a nested message dict of same format as self.msg returns. """ if isinstance(socket, ZMQStream): socket = socket.socket try: msg_list = socket.recv_multipart(mode, copy=copy) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: # We can convert EAGAIN to None as we know in this case # recv_multipart won't return None. return None,None else: raise # split multipart message into identity list and message dict # invalid large messages can cause very expensive string comparisons idents, msg_list = self.feed_identities(msg_list, copy) try: return idents, self.deserialize(msg_list, content=content, copy=copy) except Exception as e: # TODO: handle it raise e def feed_identities(self, msg_list, copy=True): """Split the identities from the rest of the message. Feed until DELIM is reached, then return the prefix as idents and remainder as msg_list. This is easily broken by setting an IDENT to DELIM, but that would be silly. Parameters ---------- msg_list : a list of Message or bytes objects The message to be split. copy : bool flag determining whether the arguments are bytes or Messages Returns ------- (idents, msg_list) : two lists idents will always be a list of bytes, each of which is a ZMQ identity. msg_list will be a list of bytes or zmq.Messages of the form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and should be unpackable/unserializable via self.deserialize at this point. """ if copy: idx = msg_list.index(DELIM) return msg_list[:idx], msg_list[idx+1:] else: failed = True for idx,m in enumerate(msg_list): if m.bytes == DELIM: failed = False break if failed: raise ValueError("DELIM not in msg_list") idents, msg_list = msg_list[:idx], msg_list[idx+1:] return [m.bytes for m in idents], msg_list def _add_digest(self, signature): """add a digest to history to protect against replay attacks""" if self.digest_history_size == 0: # no history, never add digests return self.digest_history.add(signature) if len(self.digest_history) > self.digest_history_size: # threshold reached, cull 10% self._cull_digest_history() def _cull_digest_history(self): """cull the digest history Removes a randomly selected 10% of the digest history """ current = len(self.digest_history) n_to_cull = max(int(current // 10), current - self.digest_history_size) if n_to_cull >= current: self.digest_history = set() return to_cull = random.sample(self.digest_history, n_to_cull) self.digest_history.difference_update(to_cull) def deserialize(self, msg_list, content=True, copy=True): """Unserialize a msg_list to a nested message dict. This is roughly the inverse of serialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg_list : list of bytes or Message objects The list of message parts of the form [HMAC,p_header,p_parent, p_metadata,p_content,buffer1,buffer2,...]. content : bool (True) Whether to unpack the content dict (True), or leave it packed (False). copy : bool (True) Whether msg_list contains bytes (True) or the non-copying Message objects in each place (False). Returns ------- msg : dict The nested message dict with top-level keys [header, parent_header, content, buffers]. The buffers are returned as memoryviews. """ minlen = 5 message = {} if not copy: # pyzmq didn't copy the first parts of the message, so we'll do it for i in range(minlen): msg_list[i] = msg_list[i].bytes if self.auth is not None: signature = msg_list[0] if not signature: raise ValueError("Unsigned Message") if signature in self.digest_history: raise ValueError("Duplicate Signature: %r" % signature) if content: # Only store signature if we are unpacking content, don't store if just peeking. self._add_digest(signature) check = self.sign(msg_list[1:5]) if not compare_digest(signature, check): raise ValueError("Invalid Signature: %r" % signature) if not len(msg_list) >= minlen: raise TypeError("malformed message, must have at least %i elements"%minlen) header = self.unpack(msg_list[1]) message['header'] = extract_dates(header) message['msg_id'] = header['msg_id'] message['msg_type'] = header['msg_type'] message['parent_header'] = extract_dates(self.unpack(msg_list[2])) message['metadata'] = self.unpack(msg_list[3]) if content: message['content'] = self.unpack(msg_list[4]) else: message['content'] = msg_list[4] buffers = [memoryview(b) for b in msg_list[5:]] if buffers and buffers[0].shape is None: # force copy to workaround pyzmq #646 buffers = [memoryview(b.bytes) for b in msg_list[5:]] message['buffers'] = buffers if self.debug: pprint.pprint(message) # adapt to the current version return adapt(message) def unserialize(self, *args, **kwargs): warnings.warn( "Session.unserialize is deprecated. Use Session.deserialize.", DeprecationWarning, ) return self.deserialize(*args, **kwargs)
class KernelManager(ConnectionFileMixin): """Manages a single kernel in a subprocess on this host. This version starts kernels with Popen. """ _created_context = Bool(False) # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.Context) def _context_default(self): self._created_context = True return zmq.Context() # the class to create with our `client` method client_class = DottedObjectName( 'jupyter_client.blocking.BlockingKernelClient') client_factory = Type(klass='jupyter_client.KernelClient') def _client_factory_default(self): return import_item(self.client_class) @observe('client_class') def _client_class_changed(self, change): self.client_factory = import_item(str(change['new'])) # The kernel process with which the KernelManager is communicating. # generally a Popen instance kernel = Any() kernel_spec_manager = Instance(kernelspec.KernelSpecManager) def _kernel_spec_manager_default(self): return kernelspec.KernelSpecManager(data_dir=self.data_dir) @observe('kernel_spec_manager') @observe_compat def _kernel_spec_manager_changed(self, change): self._kernel_spec = None shutdown_wait_time = Float( 5.0, config=True, help="Time to wait for a kernel to terminate before killing it, " "in seconds.") kernel_name = Unicode(kernelspec.NATIVE_KERNEL_NAME) @observe('kernel_name') def _kernel_name_changed(self, change): self._kernel_spec = None if change['new'] == 'python': self.kernel_name = kernelspec.NATIVE_KERNEL_NAME _kernel_spec = None @property def kernel_spec(self): if self._kernel_spec is None and self.kernel_name != '': self._kernel_spec = self.kernel_spec_manager.get_kernel_spec( self.kernel_name) return self._kernel_spec kernel_cmd = List(Unicode(), config=True, help="""DEPRECATED: Use kernel_name instead. The Popen Command to launch the kernel. Override this if you have a custom kernel. If kernel_cmd is specified in a configuration file, Jupyter does not pass any arguments to the kernel, because it cannot make any assumptions about the arguments that the kernel understands. In particular, this means that the kernel does not receive the option --debug if it given on the Jupyter command line. """) def _kernel_cmd_changed(self, name, old, new): warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to " "start different kernels.") cache_ports = Bool( help= 'True if the MultiKernelManager should cache ports for this KernelManager instance' ) @default('cache_ports') def _default_cache_ports(self): return self.transport == 'tcp' @property def ipykernel(self): return self.kernel_name in {'python', 'python2', 'python3'} # Protected traits _launch_args = Any() _control_socket = Any() _restarter = Any() autorestart = Bool(True, config=True, help="""Should we autorestart the kernel if it dies.""") def __del__(self): self._close_control_socket() self.cleanup_connection_file() #-------------------------------------------------------------------------- # Kernel restarter #-------------------------------------------------------------------------- def start_restarter(self): pass def stop_restarter(self): pass def add_restart_callback(self, callback, event='restart'): """register a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.add_callback(callback, event) def remove_restart_callback(self, callback, event='restart'): """unregister a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.remove_callback(callback, event) #-------------------------------------------------------------------------- # create a Client connected to our Kernel #-------------------------------------------------------------------------- def client(self, **kwargs): """Create a client configured to connect to our kernel""" kw = {} kw.update(self.get_connection_info(session=True)) kw.update(dict( connection_file=self.connection_file, parent=self, )) # add kwargs last, for manual overrides kw.update(kwargs) return self.client_factory(**kw) #-------------------------------------------------------------------------- # Kernel management #-------------------------------------------------------------------------- def format_kernel_cmd(self, extra_arguments=None): """replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] if self.kernel_cmd: cmd = self.kernel_cmd + extra_arguments else: cmd = self.kernel_spec.argv + extra_arguments if cmd and cmd[0] in { 'python', 'python%i' % sys.version_info[0], 'python%i.%i' % sys.version_info[:2] }: # executable is 'python' or 'python3', use sys.executable. # These will typically be the same, # but if the current process is in an env # and has been launched by abspath without # activating the env, python on PATH may not be sys.executable, # but it should be. cmd[0] = sys.executable # Make sure to use the realpath for the connection_file # On windows, when running with the store python, the connection_file path # is not usable by non python kernels because the path is being rerouted when # inside of a store app. # See this bug here: https://bugs.python.org/issue41196 ns = dict( connection_file=os.path.realpath(self.connection_file), prefix=sys.prefix, ) if self.kernel_spec: ns["resource_dir"] = self.kernel_spec.resource_dir ns.update(self._launch_args) pat = re.compile(r'\{([A-Za-z0-9_]+)\}') def from_ns(match): """Get the key out of ns if it's there, otherwise no change.""" return ns.get(match.group(1), match.group()) return [pat.sub(from_ns, arg) for arg in cmd] def _launch_kernel(self, kernel_cmd, **kw): """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ return launch_kernel(kernel_cmd, **kw) # Control socket used for polite kernel shutdown def _connect_control_socket(self): if self._control_socket is None: self._control_socket = self._create_connected_socket('control') self._control_socket.linger = 100 def _close_control_socket(self): if self._control_socket is None: return self._control_socket.close() self._control_socket = None def pre_start_kernel(self, **kw): """Prepares a kernel for startup in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError( "Can only launch a kernel on a local interface. " "This one is not: %s." "Make sure that the '*_address' attributes are " "configured properly. " "Currently valid addresses are: %s" % (self.ip, local_ips())) # write connection file / get default ports self.write_connection_file() # save kwargs for use in restart self._launch_args = kw.copy() # build the Popen cmd extra_arguments = kw.pop('extra_arguments', []) kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) env = kw.pop('env', os.environ).copy() # Don't allow PYTHONEXECUTABLE to be passed to kernel process. # If set, it can bork all the things. env.pop('PYTHONEXECUTABLE', None) if not self.kernel_cmd: # If kernel_cmd has been set manually, don't refer to a kernel spec. # Environment variables from kernel spec are added to os.environ. env.update(self._get_env_substitutions(self.kernel_spec.env, env)) kw['env'] = env return kernel_cmd, kw def _get_env_substitutions(self, templated_env, substitution_values): """ Walks env entries in templated_env and applies possible substitutions from current env (represented by substitution_values). Returns the substituted list of env entries. """ substituted_env = {} if templated_env: from string import Template # For each templated env entry, fill any templated references # matching names of env variables with those values and build # new dict with substitutions. for k, v in templated_env.items(): substituted_env.update( {k: Template(v).safe_substitute(substitution_values)}) return substituted_env def post_start_kernel(self, **kw): self.start_restarter() self._connect_control_socket() def start_kernel(self, **kw): """Starts a kernel on this host in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ kernel_cmd, kw = self.pre_start_kernel(**kw) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) self.kernel = self._launch_kernel(kernel_cmd, **kw) self.post_start_kernel(**kw) def request_shutdown(self, restart=False): """Send a shutdown request via control channel """ content = dict(restart=restart) msg = self.session.msg("shutdown_request", content=content) # ensure control socket is connected self._connect_control_socket() self.session.send(self._control_socket, msg) def finish_shutdown(self, waittime=None, pollinterval=0.1): """Wait for kernel shutdown, then kill process if it doesn't shutdown. This does not send shutdown requests - use :meth:`request_shutdown` first. """ if waittime is None: waittime = max(self.shutdown_wait_time, 0) for i in range(int(waittime / pollinterval)): if self.is_alive(): time.sleep(pollinterval) else: # If there's still a proc, wait and clear if self.has_kernel: self.kernel.wait() self.kernel = None break else: # OK, we've waited long enough. if self.has_kernel: self.log.debug("Kernel is taking too long to finish, killing") self._kill_kernel() def cleanup_resources(self, restart=False): """Clean up resources when the kernel is shut down""" if not restart: self.cleanup_connection_file() self.cleanup_ipc_files() self._close_control_socket() self.session.parent = None if self._created_context and not restart: self.context.destroy(linger=100) def cleanup(self, connection_file=True): """Clean up resources when the kernel is shut down""" warnings.warn( "Method cleanup(connection_file=True) is deprecated, use cleanup_resources(restart=False).", FutureWarning) self.cleanup_resources(restart=not connection_file) def shutdown_kernel(self, now=False, restart=False): """Attempts to stop the kernel process cleanly. This attempts to shutdown the kernels cleanly by: 1. Sending it a shutdown message over the control channel. 2. If that fails, the kernel is shutdown forcibly by sending it a signal. Parameters ---------- now : bool Should the kernel be forcible killed *now*. This skips the first, nice shutdown attempt. restart: bool Will this kernel be restarted after it is shutdown. When this is True, connection files will not be cleaned up. """ # Stop monitoring for restarting while we shutdown. self.stop_restarter() if now: self._kill_kernel() else: self.request_shutdown(restart=restart) # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. self.finish_shutdown() # In 6.1.5, a new method, cleanup_resources(), was introduced to address # a leak issue (https://github.com/jupyter/jupyter_client/pull/548) and # replaced the existing cleanup() method. However, that method introduction # breaks subclass implementations that override cleanup() since it would # circumvent cleanup() functionality implemented in subclasses. # By detecting if the current instance overrides cleanup(), we can determine # if the deprecated path of calling cleanup() should be performed - which avoids # unnecessary deprecation warnings in a majority of configurations in which # subclassed KernelManager instances are not in use. # Note: because subclasses may have already implemented cleanup_resources() # but need to support older jupyter_clients, we should only take the deprecated # path if cleanup() is overridden but cleanup_resources() is not. overrides_cleanup = type(self).cleanup is not KernelManager.cleanup overrides_cleanup_resources = type( self).cleanup_resources is not KernelManager.cleanup_resources if overrides_cleanup and not overrides_cleanup_resources: self.cleanup(connection_file=not restart) else: self.cleanup_resources(restart=restart) def restart_kernel(self, now=False, newports=False, **kw): """Restarts a kernel with the arguments that were used to launch it. Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. newports : bool, optional If the old kernel was launched with random ports, this flag decides whether the same ports and connection file will be used again. If False, the same ports and connection file are used. This is the default. If True, new random port numbers are chosen and a new connection file is written. It is still possible that the newly chosen random port numbers happen to be the same as the old ones. `**kw` : optional Any options specified here will overwrite those used to launch the kernel. """ if self._launch_args is None: raise RuntimeError("Cannot restart the kernel. " "No previous call to 'start_kernel'.") else: # Stop currently running kernel. self.shutdown_kernel(now=now, restart=True) if newports: self.cleanup_random_ports() # Start new kernel. self._launch_args.update(kw) self.start_kernel(**self._launch_args) @property def has_kernel(self): """Has a kernel been started that we are managing.""" return self.kernel is not None def _kill_kernel(self): """Kill the running kernel. This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: if hasattr(signal, 'SIGKILL'): self.signal_kernel(signal.SIGKILL) else: self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. if sys.platform == 'win32': if e.winerror != 5: raise # On Unix, we may get an ESRCH error if the process has already # terminated. Ignore it. else: from errno import ESRCH if e.errno != ESRCH: raise # Block until the kernel terminates. self.kernel.wait() self.kernel = None def interrupt_kernel(self): """Interrupts the kernel by sending it a signal. Unlike ``signal_kernel``, this operation is well supported on all platforms. """ if self.has_kernel: interrupt_mode = self.kernel_spec.interrupt_mode if interrupt_mode == 'signal': if sys.platform == 'win32': from .win_interrupt import send_interrupt send_interrupt(self.kernel.win32_interrupt_event) else: self.signal_kernel(signal.SIGINT) elif interrupt_mode == 'message': msg = self.session.msg("interrupt_request", content={}) self._connect_control_socket() self.session.send(self._control_socket, msg) else: raise RuntimeError( "Cannot interrupt kernel. No kernel is running!") def signal_kernel(self, signum): """Sends a signal to the process group of the kernel (this usually includes the kernel and any subprocesses spawned by the kernel). Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ if self.has_kernel: if hasattr(os, "getpgid") and hasattr(os, "killpg"): try: pgid = os.getpgid(self.kernel.pid) os.killpg(pgid, signum) return except OSError: pass self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") def is_alive(self): """Is the kernel process still running?""" if self.has_kernel: if self.kernel.poll() is None: return True else: return False else: # we don't have a kernel return False
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin): """ A Qt frontend for a generic Python kernel. """ # The text to show when the kernel is (re)started. banner = Unicode(config=True) kernel_banner = Unicode() # Whether to show the banner _display_banner = Bool(False) # An option and corresponding signal for overriding the default kernel # interrupt behavior. custom_interrupt = Bool(False) custom_interrupt_requested = QtCore.Signal() # An option and corresponding signals for overriding the default kernel # restart behavior. custom_restart = Bool(False) custom_restart_kernel_died = QtCore.Signal(float) custom_restart_requested = QtCore.Signal() # Whether to automatically show calltips on open-parentheses. enable_calltips = Bool( True, config=True, help="Whether to draw information calltips on open-parentheses.") clear_on_kernel_restart = Bool( True, config=True, help="Whether to clear the console when the kernel is restarted") confirm_restart = Bool( True, config=True, help="Whether to ask for user confirmation when restarting kernel") lexer_class = DottedObjectName(config=True, help="The pygments lexer class to use.") def _lexer_class_changed(self, name, old, new): lexer_class = import_item(new) self.lexer = lexer_class() def _lexer_class_default(self): if py3compat.PY3: return 'pygments.lexers.Python3Lexer' else: return 'pygments.lexers.PythonLexer' lexer = Any() def _lexer_default(self): lexer_class = import_item(self.lexer_class) return lexer_class() # Emitted when a user visible 'execute_request' has been submitted to the # kernel from the FrontendWidget. Contains the code to be executed. executing = QtCore.Signal(object) # Emitted when a user-visible 'execute_reply' has been received from the # kernel and processed by the FrontendWidget. Contains the response message. executed = QtCore.Signal(object) # Emitted when an exit request has been received from the kernel. exit_requested = QtCore.Signal(object) _CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos']) _CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos']) _ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind']) _local_kernel = False _highlighter = Instance(FrontendHighlighter, allow_none=True) #--------------------------------------------------------------------------- # 'object' interface #--------------------------------------------------------------------------- def __init__(self, *args, **kw): super(FrontendWidget, self).__init__(*args, **kw) # FIXME: remove this when PySide min version is updated past 1.0.7 # forcefully disable calltips if PySide is < 1.0.7, because they crash if qt.QT_API == qt.QT_API_PYSIDE: import PySide if PySide.__version_info__ < (1, 0, 7): self.log.warn( "PySide %s < 1.0.7 detected, disabling calltips" % PySide.__version__) self.enable_calltips = False # FrontendWidget protected variables. self._bracket_matcher = BracketMatcher(self._control) self._call_tip_widget = CallTipWidget(self._control) self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None) self._hidden = False self._highlighter = FrontendHighlighter(self, lexer=self.lexer) self._kernel_manager = None self._kernel_client = None self._request_info = {} self._request_info['execute'] = {} self._callback_dict = {} self._display_banner = True # Configure the ConsoleWidget. self.tab_width = 4 self._set_continuation_prompt('... ') # Configure the CallTipWidget. self._call_tip_widget.setFont(self.font) self.font_changed.connect(self._call_tip_widget.setFont) # Configure actions. action = self._copy_raw_action key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C action.setEnabled(False) action.setShortcut(QtGui.QKeySequence(key)) action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut) action.triggered.connect(self.copy_raw) self.copy_available.connect(action.setEnabled) self.addAction(action) # Connect signal handlers. document = self._control.document() document.contentsChange.connect(self._document_contents_change) # Set flag for whether we are connected via localhost. self._local_kernel = kw.get('local_kernel', FrontendWidget._local_kernel) # Whether or not a clear_output call is pending new output. self._pending_clearoutput = False #--------------------------------------------------------------------------- # 'ConsoleWidget' public interface #--------------------------------------------------------------------------- def copy(self): """ Copy the currently selected text to the clipboard, removing prompts. """ if self._page_control is not None and self._page_control.hasFocus(): self._page_control.copy() elif self._control.hasFocus(): text = self._control.textCursor().selection().toPlainText() if text: # Remove prompts. lines = text.splitlines() lines = map(self._highlighter.transform_classic_prompt, lines) lines = map(self._highlighter.transform_ipy_prompt, lines) text = '\n'.join(lines) # Needed to prevent errors when copying the prompt. # See issue 264 try: was_newline = text[-1] == '\n' except IndexError: was_newline = False if was_newline: # user doesn't need newline text = text[:-1] QtGui.QApplication.clipboard().setText(text) else: self.log.debug("frontend widget : unknown copy target") #--------------------------------------------------------------------------- # 'ConsoleWidget' abstract interface #--------------------------------------------------------------------------- def _execute(self, source, hidden): """ Execute 'source'. If 'hidden', do not show any output. See parent class :meth:`execute` docstring for full details. """ msg_id = self.kernel_client.execute(source, hidden) self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'user') self._hidden = hidden if not hidden: self.executing.emit(source) def _prompt_started_hook(self): """ Called immediately after a new prompt is displayed. """ if not self._reading: self._highlighter.highlighting_on = True def _prompt_finished_hook(self): """ Called immediately after a prompt is finished, i.e. when some input will be processed and a new prompt displayed. """ if not self._reading: self._highlighter.highlighting_on = False def _tab_pressed(self): """ Called when the tab key is pressed. Returns whether to continue processing the event. """ # Perform tab completion if: # 1) The cursor is in the input buffer. # 2) There is a non-whitespace character before the cursor. # 3) There is no active selection. text = self._get_input_buffer_cursor_line() if text is None: return False non_ws_before = bool( text[:self._get_input_buffer_cursor_column()].strip()) complete = non_ws_before and self._get_cursor().selectedText() == '' if complete: self._complete() return not complete #--------------------------------------------------------------------------- # 'ConsoleWidget' protected interface #--------------------------------------------------------------------------- def _context_menu_make(self, pos): """ Reimplemented to add an action for raw copy. """ menu = super(FrontendWidget, self)._context_menu_make(pos) for before_action in menu.actions(): if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \ QtGui.QKeySequence.ExactMatch: menu.insertAction(before_action, self._copy_raw_action) break return menu def request_interrupt_kernel(self): if self._executing: self.interrupt_kernel() def request_restart_kernel(self): message = 'Are you sure you want to restart the kernel?' self.restart_kernel(message, now=False) def _event_filter_console_keypress(self, event): """ Reimplemented for execution interruption and smart backspace. """ key = event.key() if self._control_key_down(event.modifiers(), include_command=False): if key == QtCore.Qt.Key_C and self._executing: self.request_interrupt_kernel() return True elif key == QtCore.Qt.Key_Period: self.request_restart_kernel() return True elif not event.modifiers() & QtCore.Qt.AltModifier: # Smart backspace: remove four characters in one backspace if: # 1) everything left of the cursor is whitespace # 2) the four characters immediately left of the cursor are spaces if key == QtCore.Qt.Key_Backspace: col = self._get_input_buffer_cursor_column() cursor = self._control.textCursor() if col > 3 and not cursor.hasSelection(): text = self._get_input_buffer_cursor_line()[:col] if text.endswith(' ') and not text.strip(): cursor.movePosition(QtGui.QTextCursor.Left, QtGui.QTextCursor.KeepAnchor, 4) cursor.removeSelectedText() return True return super(FrontendWidget, self)._event_filter_console_keypress(event) #--------------------------------------------------------------------------- # 'BaseFrontendMixin' abstract interface #--------------------------------------------------------------------------- def _handle_clear_output(self, msg): """Handle clear output messages.""" if self.include_output(msg): wait = msg['content'].get('wait', True) if wait: self._pending_clearoutput = True else: self.clear_output() def _silent_exec_callback(self, expr, callback): """Silently execute `expr` in the kernel and call `callback` with reply the `expr` is evaluated silently in the kernel (without) output in the frontend. Call `callback` with the `repr <http://docs.python.org/library/functions.html#repr> `_ as first argument Parameters ---------- expr : string valid string to be executed by the kernel. callback : function function accepting one argument, as a string. The string will be the `repr` of the result of evaluating `expr` The `callback` is called with the `repr()` of the result of `expr` as first argument. To get the object, do `eval()` on the passed value. See Also -------- _handle_exec_callback : private method, deal with calling callback with reply """ # generate uuid, which would be used as an indication of whether or # not the unique request originated from here (can use msg id ?) local_uuid = str(uuid.uuid1()) msg_id = self.kernel_client.execute( '', silent=True, user_expressions={local_uuid: expr}) self._callback_dict[local_uuid] = callback self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'silent_exec_callback') def _handle_exec_callback(self, msg): """Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback`` Parameters ---------- msg : raw message send by the kernel containing an `user_expressions` and having a 'silent_exec_callback' kind. Notes ----- This function will look for a `callback` associated with the corresponding message id. Association has been made by `_silent_exec_callback`. `callback` is then called with the `repr()` of the value of corresponding `user_expressions` as argument. `callback` is then removed from the known list so that any message coming again with the same id won't trigger it. """ user_exp = msg['content'].get('user_expressions') if not user_exp: return for expression in user_exp: if expression in self._callback_dict: self._callback_dict.pop(expression)(user_exp[expression]) def _handle_execute_reply(self, msg): """ Handles replies for code execution. """ self.log.debug("execute_reply: %s", msg.get('content', '')) msg_id = msg['parent_header']['msg_id'] info = self._request_info['execute'].get(msg_id) # unset reading flag, because if execute finished, raw_input can't # still be pending. self._reading = False # Note: If info is NoneType, this is ignored if info and info.kind == 'user' and not self._hidden: # Make sure that all output from the SUB channel has been processed # before writing a new prompt. self.kernel_client.iopub_channel.flush() # Reset the ANSI style information to prevent bad text in stdout # from messing up our colors. We're not a true terminal so we're # allowed to do this. if self.ansi_codes: self._ansi_processor.reset_sgr() content = msg['content'] status = content['status'] if status == 'ok': self._process_execute_ok(msg) elif status == 'aborted': self._process_execute_abort(msg) self._show_interpreter_prompt_for_reply(msg) self.executed.emit(msg) self._request_info['execute'].pop(msg_id) elif info and info.kind == 'silent_exec_callback' and not self._hidden: self._handle_exec_callback(msg) self._request_info['execute'].pop(msg_id) elif info and not self._hidden: raise RuntimeError("Unknown handler for %s" % info.kind) def _handle_error(self, msg): """ Handle error messages. """ self._process_execute_error(msg) def _handle_input_request(self, msg): """ Handle requests for raw_input. """ self.log.debug("input: %s", msg.get('content', '')) if self._hidden: raise RuntimeError( 'Request for raw input during hidden execution.') # Make sure that all output from the SUB channel has been processed # before entering readline mode. self.kernel_client.iopub_channel.flush() def callback(line): self.kernel_client.input(line) if self._reading: self.log.debug( "Got second input request, assuming first was interrupted.") self._reading = False self._readline(msg['content']['prompt'], callback=callback, password=msg['content']['password']) def _kernel_restarted_message(self, died=True): msg = "Kernel died, restarting" if died else "Kernel restarting" self._append_html("<br>%s<hr><br>" % msg, before_prompt=False) def _handle_kernel_died(self, since_last_heartbeat): """Handle the kernel's death (if we do not own the kernel). """ self.log.warn("kernel died: %s", since_last_heartbeat) if self.custom_restart: self.custom_restart_kernel_died.emit(since_last_heartbeat) else: self._kernel_restarted_message(died=True) self.reset() def _handle_kernel_restarted(self, died=True): """Notice that the autorestarter restarted the kernel. There's nothing to do but show a message. """ self.log.warn("kernel restarted") self._kernel_restarted_message(died=died) self.reset() def _handle_inspect_reply(self, rep): """Handle replies for call tips.""" self.log.debug("oinfo: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('call_tip') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): content = rep['content'] if content.get('status') == 'ok' and content.get('found', False): self._call_tip_widget.show_inspect_data(content) def _handle_execute_result(self, msg): """ Handle display hook output. """ self.log.debug("execute_result: %s", msg.get('content', '')) if self.include_output(msg): self.flush_clearoutput() text = msg['content']['data'] self._append_plain_text(text + '\n', before_prompt=True) def _handle_stream(self, msg): """ Handle stdout, stderr, and stdin. """ self.log.debug("stream: %s", msg.get('content', '')) if self.include_output(msg): self.flush_clearoutput() self.append_stream(msg['content']['text']) def _handle_shutdown_reply(self, msg): """ Handle shutdown signal, only if from other console. """ self.log.debug("shutdown: %s", msg.get('content', '')) restart = msg.get('content', {}).get('restart', False) if not self._hidden and not self.from_here(msg): # got shutdown reply, request came from session other than ours if restart: # someone restarted the kernel, handle it self._handle_kernel_restarted(died=False) else: # kernel was shutdown permanently # this triggers exit_requested if the kernel was local, # and a dialog if the kernel was remote, # so we don't suddenly clear the qtconsole without asking. if self._local_kernel: self.exit_requested.emit(self) else: title = self.window().windowTitle() reply = QtGui.QMessageBox.question( self, title, "Kernel has been shutdown permanently. " "Close the Console?", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No) if reply == QtGui.QMessageBox.Yes: self.exit_requested.emit(self) def _handle_status(self, msg): """Handle status message""" # This is where a busy/idle indicator would be triggered, # when we make one. state = msg['content'].get('execution_state', '') if state == 'starting': # kernel started while we were running if self._executing: self._handle_kernel_restarted(died=True) elif state == 'idle': pass elif state == 'busy': pass def _started_channels(self): """ Called when the KernelManager channels have started listening or when the frontend is assigned an already listening KernelManager. """ self.reset(clear=True) #--------------------------------------------------------------------------- # 'FrontendWidget' public interface #--------------------------------------------------------------------------- def copy_raw(self): """ Copy the currently selected text to the clipboard without attempting to remove prompts or otherwise alter the text. """ self._control.copy() def interrupt_kernel(self): """ Attempts to interrupt the running kernel. Also unsets _reading flag, to avoid runtime errors if raw_input is called again. """ if self.custom_interrupt: self._reading = False self.custom_interrupt_requested.emit() elif self.kernel_manager: self._reading = False self.kernel_manager.interrupt_kernel() else: self._append_plain_text( 'Cannot interrupt a kernel I did not start.\n') def reset(self, clear=False): """ Resets the widget to its initial state if ``clear`` parameter is True, otherwise prints a visual indication of the fact that the kernel restarted, but does not clear the traces from previous usage of the kernel before it was restarted. With ``clear=True``, it is similar to ``%clear``, but also re-writes the banner and aborts execution if necessary. """ if self._executing: self._executing = False self._request_info['execute'] = {} self._reading = False self._highlighter.highlighting_on = False if clear: self._control.clear() if self._display_banner: self._append_plain_text(self.banner) if self.kernel_banner: self._append_plain_text(self.kernel_banner) # update output marker for stdout/stderr, so that startup # messages appear after banner: self._show_interpreter_prompt() def restart_kernel(self, message, now=False): """ Attempts to restart the running kernel. """ # FIXME: now should be configurable via a checkbox in the dialog. Right # now at least the heartbeat path sets it to True and the manual restart # to False. But those should just be the pre-selected states of a # checkbox that the user could override if so desired. But I don't know # enough Qt to go implementing the checkbox now. if self.custom_restart: self.custom_restart_requested.emit() return if self.kernel_manager: # Pause the heart beat channel to prevent further warnings. self.kernel_client.hb_channel.pause() # Prompt the user to restart the kernel. Un-pause the heartbeat if # they decline. (If they accept, the heartbeat will be un-paused # automatically when the kernel is restarted.) if self.confirm_restart: buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No result = QtGui.QMessageBox.question(self, 'Restart kernel?', message, buttons) do_restart = result == QtGui.QMessageBox.Yes else: # confirm_restart is False, so we don't need to ask user # anything, just do the restart do_restart = True if do_restart: try: self.kernel_manager.restart_kernel(now=now) except RuntimeError as e: self._append_plain_text('Error restarting kernel: %s\n' % e, before_prompt=True) else: self._append_html( "<br>Restarting kernel...\n<hr><br>", before_prompt=True, ) else: self.kernel_client.hb_channel.unpause() else: self._append_plain_text( 'Cannot restart a Kernel I did not start\n', before_prompt=True) def append_stream(self, text): """Appends text to the output stream.""" # Most consoles treat tabs as being 8 space characters. Convert tabs # to spaces so that output looks as expected regardless of this # widget's tab width. text = text.expandtabs(8) self._append_plain_text(text, before_prompt=True) self._control.moveCursor(QtGui.QTextCursor.End) def flush_clearoutput(self): """If a clearoutput is pending, execute it.""" if self._pending_clearoutput: self._pending_clearoutput = False self.clear_output() def clear_output(self): """Clears the current line of output.""" cursor = self._control.textCursor() cursor.beginEditBlock() cursor.movePosition(cursor.StartOfLine, cursor.KeepAnchor) cursor.insertText('') cursor.endEditBlock() #--------------------------------------------------------------------------- # 'FrontendWidget' protected interface #--------------------------------------------------------------------------- def _auto_call_tip(self): """Trigger call tip automatically on open parenthesis Call tips can be requested explcitly with `_call_tip`. """ cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.Left) if cursor.document().characterAt(cursor.position()) == '(': # trigger auto call tip on open paren self._call_tip() def _call_tip(self): """Shows a call tip, if appropriate, at the current cursor location.""" # Decide if it makes sense to show a call tip if not self.enable_calltips or not self.kernel_client.shell_channel.is_alive( ): return False cursor_pos = self._get_input_buffer_cursor_pos() code = self.input_buffer # Send the metadata request to the kernel msg_id = self.kernel_client.inspect(code, cursor_pos) pos = self._get_cursor().position() self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos) return True def _complete(self): """ Performs completion at the current cursor location. """ # Send the completion request to the kernel msg_id = self.kernel_client.complete( code=self.input_buffer, cursor_pos=self._get_input_buffer_cursor_pos(), ) pos = self._get_cursor().position() info = self._CompletionRequest(msg_id, pos) self._request_info['complete'] = info def _process_execute_abort(self, msg): """ Process a reply for an aborted execution request. """ self._append_plain_text("ERROR: execution aborted\n") def _process_execute_error(self, msg): """ Process a reply for an execution request that resulted in an error. """ content = msg['content'] # If a SystemExit is passed along, this means exit() was called - also # all the ipython %exit magic syntax of '-k' to be used to keep # the kernel running if content['ename'] == 'SystemExit': keepkernel = content['evalue'] == '-k' or content[ 'evalue'] == 'True' self._keep_kernel_on_exit = keepkernel self.exit_requested.emit(self) else: traceback = ''.join(content['traceback']) self._append_plain_text(traceback) def _process_execute_ok(self, msg): """ Process a reply for a successful execution request. """ payload = msg['content'].get('payload', []) for item in payload: if not self._process_execute_payload(item): warning = 'Warning: received unknown payload of type %s' print(warning % repr(item['source'])) def _process_execute_payload(self, item): """ Process a single payload item from the list of payload items in an execution reply. Returns whether the payload was handled. """ # The basic FrontendWidget doesn't handle payloads, as they are a # mechanism for going beyond the standard Python interpreter model. return False def _show_interpreter_prompt(self): """ Shows a prompt for the interpreter. """ self._show_prompt('>>> ') def _show_interpreter_prompt_for_reply(self, msg): """ Shows a prompt for the interpreter given an 'execute_reply' message. """ self._show_interpreter_prompt() #------ Signal handlers ---------------------------------------------------- def _document_contents_change(self, position, removed, added): """ Called whenever the document's content changes. Display a call tip if appropriate. """ # Calculate where the cursor should be *after* the change: position += added document = self._control.document() if position == self._get_cursor().position(): self._auto_call_tip() #------ Trait default initializers ----------------------------------------- def _banner_default(self): """ Returns the standard Python banner. """ banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \ '"license" for more information.' return banner % (sys.version, sys.platform)
class MultiKernelManager(LoggingConfigurable): """A class for managing multiple kernels.""" default_kernel_name = Unicode( NATIVE_KERNEL_NAME, config=True, help="The name of the default kernel to start") kernel_spec_manager = Instance(KernelSpecManager, allow_none=True) kernel_manager_class = DottedObjectName( "jupyter_client.ioloop.IOLoopKernelManager", config=True, help="""The kernel manager class. This is configurable to allow subclassing of the KernelManager for customized behavior. """) def _kernel_manager_class_changed(self, name, old, new): self.kernel_manager_factory = import_item(new) kernel_manager_factory = Any( help="this is kernel_manager_class after import") def _kernel_manager_factory_default(self): return import_item(self.kernel_manager_class) context = Instance('zmq.Context') def _context_default(self): return zmq.Context.instance() connection_dir = Unicode('') _kernels = Dict() def list_kernel_ids(self): """Return a list of the kernel ids of the active kernels.""" # Create a copy so we can iterate over kernels in operations # that delete keys. return list(self._kernels.keys()) def __len__(self): """Return the number of running kernels.""" return len(self.list_kernel_ids()) def __contains__(self, kernel_id): return kernel_id in self._kernels def start_kernel(self, kernel_name=None, **kwargs): """Start a new kernel. The caller can pick a kernel_id by passing one in as a keyword arg, otherwise one will be picked using a uuid. The kernel ID for the newly started kernel is returned. """ kernel_id = kwargs.pop('kernel_id', unicode_type(uuid.uuid4())) if kernel_id in self: raise DuplicateKernelError('Kernel already exists: %s' % kernel_id) if kernel_name is None: kernel_name = self.default_kernel_name # kernel_manager_factory is the constructor for the KernelManager # subclass we are using. It can be configured as any Configurable, # including things like its transport and ip. constructor_kwargs = {} if self.kernel_spec_manager: constructor_kwargs[ 'kernel_spec_manager'] = self.kernel_spec_manager km = self.kernel_manager_factory(connection_file=os.path.join( self.connection_dir, "kernel-%s.json" % kernel_id), parent=self, log=self.log, kernel_name=kernel_name, **constructor_kwargs) km.start_kernel(**kwargs) self._kernels[kernel_id] = km return kernel_id @kernel_method def shutdown_kernel(self, kernel_id, now=False, restart=False): """Shutdown a kernel by its kernel uuid. Parameters ========== kernel_id : uuid The id of the kernel to shutdown. now : bool Should the kernel be shutdown forcibly using a signal. restart : bool Will the kernel be restarted? """ self.log.info("Kernel shutdown: %s" % kernel_id) self.remove_kernel(kernel_id) @kernel_method def request_shutdown(self, kernel_id, restart=False): """Ask a kernel to shut down by its kernel uuid""" @kernel_method def finish_shutdown(self, kernel_id, waittime=None, pollinterval=0.1): """Wait for a kernel to finish shutting down, and kill it if it doesn't """ self.log.info("Kernel shutdown: %s" % kernel_id) @kernel_method def cleanup(self, kernel_id, connection_file=True): """Clean up a kernel's resources""" def remove_kernel(self, kernel_id): """remove a kernel from our mapping. Mainly so that a kernel can be removed if it is already dead, without having to call shutdown_kernel. The kernel object is returned. """ return self._kernels.pop(kernel_id) def shutdown_all(self, now=False): """Shutdown all kernels.""" kids = self.list_kernel_ids() for kid in kids: self.request_shutdown(kid) for kid in kids: self.finish_shutdown(kid) self.cleanup(kid) self.remove_kernel(kid) @kernel_method def interrupt_kernel(self, kernel_id): """Interrupt (SIGINT) the kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel interrupted: %s" % kernel_id) @kernel_method def signal_kernel(self, kernel_id, signum): """Sends a signal to the kernel by its uuid. Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. Parameters ========== kernel_id : uuid The id of the kernel to signal. """ self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum)) @kernel_method def restart_kernel(self, kernel_id, now=False): """Restart a kernel by its uuid, keeping the same ports. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel restarted: %s" % kernel_id) @kernel_method def is_alive(self, kernel_id): """Is the kernel alive. This calls KernelManager.is_alive() which calls Popen.poll on the actual kernel subprocess. Parameters ========== kernel_id : uuid The id of the kernel. """ def _check_kernel_id(self, kernel_id): """check that a kernel id is valid""" if kernel_id not in self: raise KeyError("Kernel with id not found: %s" % kernel_id) def get_kernel(self, kernel_id): """Get the single KernelManager object for a kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel. """ self._check_kernel_id(kernel_id) return self._kernels[kernel_id] @kernel_method def add_restart_callback(self, kernel_id, callback, event='restart'): """add a callback for the KernelRestarter""" @kernel_method def remove_restart_callback(self, kernel_id, callback, event='restart'): """remove a callback for the KernelRestarter""" @kernel_method def get_connection_info(self, kernel_id): """Return a dictionary of connection data for a kernel. Parameters ========== kernel_id : uuid The id of the kernel. Returns ======= connection_dict : dict A dict of the information needed to connect to a kernel. This includes the ip address and the integer port numbers of the different channels (stdin_port, iopub_port, shell_port, hb_port). """ @kernel_method def connect_iopub(self, kernel_id, identity=None): """Return a zmq Socket connected to the iopub channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_shell(self, kernel_id, identity=None): """Return a zmq Socket connected to the shell channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_stdin(self, kernel_id, identity=None): """Return a zmq Socket connected to the stdin channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_hb(self, kernel_id, identity=None): """Return a zmq Socket connected to the hb channel.
class DottedObjectNameTrait(HasTraits): value = DottedObjectName("a.b")
class SpyderKernelManager(QtKernelManager): """ Spyder kernels that live in a conda environment are now properly activated with custom activation scripts located at plugins/ipythonconsole/scripts. However, on windows the batch script is terminated but not the kernel it started so this subclass overrides the `_kill_kernel` method to properly kill the started kernels by using psutil. """ client_class = DottedObjectName( 'spyder.plugins.ipythonconsole.utils.client.SpyderKernelClient') def __init__(self, *args, **kwargs): self.shutting_down = False return QtKernelManager.__init__(self, *args, **kwargs) @staticmethod async def kill_proc_tree(pid, sig=signal.SIGTERM, include_parent=True, timeout=None, on_terminate=None): """ Kill a process tree (including grandchildren) with sig and return a (gone, still_alive) tuple. "on_terminate", if specified, is a callabck function which is called as soon as a child terminates. This is an new method not present in QtKernelManager. """ assert pid != os.getpid() # Won't kill myself! # This is necessary to avoid showing an error when restarting the # kernel after it failed to start in the first place. # Fixes spyder-ide/spyder#11872 try: parent = psutil.Process(pid) except psutil.NoSuchProcess: return ([], []) children = parent.children(recursive=True) if include_parent: children.append(parent) for child_process in children: # This is necessary to avoid an error when restarting the # kernel that started a PyQt5 application in the background. # Fixes spyder-ide/spyder#13999 try: child_process.send_signal(sig) except psutil.AccessDenied: return ([], []) gone, alive = psutil.wait_procs( children, timeout=timeout, callback=on_terminate, ) return (gone, alive) async def _async_kill_kernel(self, restart: bool = False) -> None: """Kill the running kernel. Override private method of jupyter_client 7 to be able to correctly close kernel that was started via a batch/bash script for correct conda env activation. """ if self.has_kernel: assert self.provisioner is not None # This is the additional line that was added to properly # kill the kernel started by Spyder. await self.kill_proc_tree(self.provisioner.process.pid) await self.provisioner.kill(restart=restart) # Wait until the kernel terminates. import asyncio try: await asyncio.wait_for(self._async_wait(), timeout=5.0) except asyncio.TimeoutError: # Wait timed out, just log warning but continue # - not much more we can do. self.log.warning("Wait for final termination of kernel timed" " out - continuing...") pass else: # Process is no longer alive, wait and clear if self.has_kernel: await self.provisioner.wait() _kill_kernel = run_sync(_async_kill_kernel) async def _async_send_kernel_sigterm(self, restart: bool = False) -> None: """similar to _kill_kernel, but with sigterm (not sigkill), but do not block""" if self.has_kernel: assert self.provisioner is not None # This is the line that was added to properly kill kernels started # by Spyder. await self.kill_proc_tree(self.provisioner.process.pid) _send_kernel_sigterm = run_sync(_async_send_kernel_sigterm)
class KernelManager(ConnectionFileMixin): """Manages a single kernel in a subprocess on this host. This version starts kernels with Popen. """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._shutdown_status = _ShutdownStatus.Unset _created_context: Bool = Bool(False) # The PyZMQ Context to use for communication with the kernel. context: Instance = Instance(zmq.Context) @default("context") def _context_default(self) -> zmq.Context: self._created_context = True return zmq.Context() # the class to create with our `client` method client_class: DottedObjectName = DottedObjectName( "jupyter_client.blocking.BlockingKernelClient" ) client_factory: Type = Type(klass="jupyter_client.KernelClient") @default("client_factory") def _client_factory_default(self) -> Type: return import_item(self.client_class) @observe("client_class") def _client_class_changed(self, change: t.Dict[str, DottedObjectName]) -> None: self.client_factory = import_item(str(change["new"])) # The kernel process with which the KernelManager is communicating. # generally a Popen instance kernel: Any = Any() kernel_spec_manager: Instance = Instance(kernelspec.KernelSpecManager) @default("kernel_spec_manager") def _kernel_spec_manager_default(self) -> kernelspec.KernelSpecManager: return kernelspec.KernelSpecManager(data_dir=self.data_dir) @observe("kernel_spec_manager") @observe_compat def _kernel_spec_manager_changed(self, change: t.Dict[str, Instance]) -> None: self._kernel_spec = None shutdown_wait_time: Float = Float( 5.0, config=True, help="Time to wait for a kernel to terminate before killing it, " "in seconds. When a shutdown request is initiated, the kernel " "will be immediately send and interrupt (SIGINT), followed" "by a shutdown_request message, after 1/2 of `shutdown_wait_time`" "it will be sent a terminate (SIGTERM) request, and finally at " "the end of `shutdown_wait_time` will be killed (SIGKILL). terminate " "and kill may be equivalent on windows.", ) kernel_name: Unicode = Unicode(kernelspec.NATIVE_KERNEL_NAME) @observe("kernel_name") def _kernel_name_changed(self, change: t.Dict[str, Unicode]) -> None: self._kernel_spec = None if change["new"] == "python": self.kernel_name = kernelspec.NATIVE_KERNEL_NAME _kernel_spec: t.Optional[kernelspec.KernelSpec] = None @property def kernel_spec(self) -> t.Optional[kernelspec.KernelSpec]: if self._kernel_spec is None and self.kernel_name != "": self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec kernel_cmd: List = List( Unicode(), config=True, help="""DEPRECATED: Use kernel_name instead. The Popen Command to launch the kernel. Override this if you have a custom kernel. If kernel_cmd is specified in a configuration file, Jupyter does not pass any arguments to the kernel, because it cannot make any assumptions about the arguments that the kernel understands. In particular, this means that the kernel does not receive the option --debug if it given on the Jupyter command line. """, ) def _kernel_cmd_changed(self, name, old, new): warnings.warn( "Setting kernel_cmd is deprecated, use kernel_spec to " "start different kernels." ) cache_ports: Bool = Bool( help="True if the MultiKernelManager should cache ports for this KernelManager instance" ) @default("cache_ports") def _default_cache_ports(self) -> bool: return self.transport == "tcp" @property def ipykernel(self) -> bool: return self.kernel_name in {"python", "python2", "python3"} # Protected traits _launch_args: Any = Any() _control_socket: Any = Any() _restarter: Any = Any() autorestart: Bool = Bool( True, config=True, help="""Should we autorestart the kernel if it dies.""" ) shutting_down: bool = False def __del__(self) -> None: self._close_control_socket() self.cleanup_connection_file() # -------------------------------------------------------------------------- # Kernel restarter # -------------------------------------------------------------------------- def start_restarter(self) -> None: pass def stop_restarter(self) -> None: pass def add_restart_callback(self, callback: t.Callable, event: str = "restart") -> None: """register a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.add_callback(callback, event) def remove_restart_callback(self, callback: t.Callable, event: str = "restart") -> None: """unregister a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.remove_callback(callback, event) # -------------------------------------------------------------------------- # create a Client connected to our Kernel # -------------------------------------------------------------------------- def client(self, **kwargs) -> KernelClient: """Create a client configured to connect to our kernel""" kw = {} kw.update(self.get_connection_info(session=True)) kw.update( dict( connection_file=self.connection_file, parent=self, ) ) # add kwargs last, for manual overrides kw.update(kwargs) return self.client_factory(**kw) # -------------------------------------------------------------------------- # Kernel management # -------------------------------------------------------------------------- def format_kernel_cmd(self, extra_arguments: t.Optional[t.List[str]] = None) -> t.List[str]: """replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] if self.kernel_cmd: cmd = self.kernel_cmd + extra_arguments else: assert self.kernel_spec is not None cmd = self.kernel_spec.argv + extra_arguments if cmd and cmd[0] in { "python", "python%i" % sys.version_info[0], "python%i.%i" % sys.version_info[:2], }: # executable is 'python' or 'python3', use sys.executable. # These will typically be the same, # but if the current process is in an env # and has been launched by abspath without # activating the env, python on PATH may not be sys.executable, # but it should be. cmd[0] = sys.executable # Make sure to use the realpath for the connection_file # On windows, when running with the store python, the connection_file path # is not usable by non python kernels because the path is being rerouted when # inside of a store app. # See this bug here: https://bugs.python.org/issue41196 ns = dict( connection_file=os.path.realpath(self.connection_file), prefix=sys.prefix, ) if self.kernel_spec: ns["resource_dir"] = self.kernel_spec.resource_dir ns.update(self._launch_args) pat = re.compile(r"\{([A-Za-z0-9_]+)\}") def from_ns(match): """Get the key out of ns if it's there, otherwise no change.""" return ns.get(match.group(1), match.group()) return [pat.sub(from_ns, arg) for arg in cmd] async def _async_launch_kernel(self, kernel_cmd: t.List[str], **kw) -> Popen: """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ return launch_kernel(kernel_cmd, **kw) _launch_kernel = run_sync(_async_launch_kernel) # Control socket used for polite kernel shutdown def _connect_control_socket(self) -> None: if self._control_socket is None: self._control_socket = self._create_connected_socket("control") self._control_socket.linger = 100 def _close_control_socket(self) -> None: if self._control_socket is None: return self._control_socket.close() self._control_socket = None def pre_start_kernel(self, **kw) -> t.Tuple[t.List[str], t.Dict[str, t.Any]]: """Prepares a kernel for startup in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ self.shutting_down = False if self.transport == "tcp" and not is_local_ip(self.ip): raise RuntimeError( "Can only launch a kernel on a local interface. " "This one is not: %s." "Make sure that the '*_address' attributes are " "configured properly. " "Currently valid addresses are: %s" % (self.ip, local_ips()) ) # write connection file / get default ports self.write_connection_file() # save kwargs for use in restart self._launch_args = kw.copy() # build the Popen cmd extra_arguments = kw.pop("extra_arguments", []) kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) env = kw.pop("env", os.environ).copy() # Don't allow PYTHONEXECUTABLE to be passed to kernel process. # If set, it can bork all the things. env.pop("PYTHONEXECUTABLE", None) if not self.kernel_cmd: # If kernel_cmd has been set manually, don't refer to a kernel spec. # Environment variables from kernel spec are added to os.environ. assert self.kernel_spec is not None env.update(self._get_env_substitutions(self.kernel_spec.env, env)) kw["env"] = env return kernel_cmd, kw def _get_env_substitutions( self, templated_env: t.Optional[t.Dict[str, str]], substitution_values: t.Dict[str, str], ) -> t.Optional[t.Dict[str, str]]: """Walks env entries in templated_env and applies possible substitutions from current env (represented by substitution_values). Returns the substituted list of env entries. """ substituted_env = {} if templated_env: from string import Template # For each templated env entry, fill any templated references # matching names of env variables with those values and build # new dict with substitutions. for k, v in templated_env.items(): substituted_env.update({k: Template(v).safe_substitute(substitution_values)}) return substituted_env def post_start_kernel(self, **kw) -> None: self.start_restarter() self._connect_control_socket() async def _async_start_kernel(self, **kw): """Starts a kernel on this host in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ kernel_cmd, kw = self.pre_start_kernel(**kw) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) self.kernel = await ensure_async(self._launch_kernel(kernel_cmd, **kw)) self.post_start_kernel(**kw) start_kernel = run_sync(_async_start_kernel) def request_shutdown(self, restart: bool = False) -> None: """Send a shutdown request via control channel""" content = dict(restart=restart) msg = self.session.msg("shutdown_request", content=content) # ensure control socket is connected self._connect_control_socket() self.session.send(self._control_socket, msg) async def _async_finish_shutdown( self, waittime: t.Optional[float] = None, pollinterval: float = 0.1 ) -> None: """Wait for kernel shutdown, then kill process if it doesn't shutdown. This does not send shutdown requests - use :meth:`request_shutdown` first. """ if waittime is None: waittime = max(self.shutdown_wait_time, 0) self._shutdown_status = _ShutdownStatus.ShutdownRequest try: await asyncio.wait_for( self._async_wait(pollinterval=pollinterval), timeout=waittime / 2 ) except asyncio.TimeoutError: self.log.debug("Kernel is taking too long to finish, terminating") self._shutdown_status = _ShutdownStatus.SigtermRequest await self._async_send_kernel_sigterm() try: await asyncio.wait_for( self._async_wait(pollinterval=pollinterval), timeout=waittime / 2 ) except asyncio.TimeoutError: self.log.debug("Kernel is taking too long to finish, killing") self._shutdown_status = _ShutdownStatus.SigkillRequest await ensure_async(self._kill_kernel()) else: # Process is no longer alive, wait and clear if self.kernel is not None: while self.kernel.poll() is None: await asyncio.sleep(pollinterval) self.kernel = None finish_shutdown = run_sync(_async_finish_shutdown) def cleanup_resources(self, restart: bool = False) -> None: """Clean up resources when the kernel is shut down""" if not restart: self.cleanup_connection_file() self.cleanup_ipc_files() self._close_control_socket() self.session.parent = None if self._created_context and not restart: self.context.destroy(linger=100) def cleanup(self, connection_file: bool = True) -> None: """Clean up resources when the kernel is shut down""" warnings.warn( "Method cleanup(connection_file=True) is deprecated, use cleanup_resources" "(restart=False).", FutureWarning, ) self.cleanup_resources(restart=not connection_file) async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False): """Attempts to stop the kernel process cleanly. This attempts to shutdown the kernels cleanly by: 1. Sending it a shutdown message over the control channel. 2. If that fails, the kernel is shutdown forcibly by sending it a signal. Parameters ---------- now : bool Should the kernel be forcible killed *now*. This skips the first, nice shutdown attempt. restart: bool Will this kernel be restarted after it is shutdown. When this is True, connection files will not be cleaned up. """ self.shutting_down = True # Used by restarter to prevent race condition # Stop monitoring for restarting while we shutdown. self.stop_restarter() await ensure_async(self.interrupt_kernel()) if now: await ensure_async(self._kill_kernel()) else: self.request_shutdown(restart=restart) # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. await ensure_async(self.finish_shutdown()) # In 6.1.5, a new method, cleanup_resources(), was introduced to address # a leak issue (https://github.com/jupyter/jupyter_client/pull/548) and # replaced the existing cleanup() method. However, that method introduction # breaks subclass implementations that override cleanup() since it would # circumvent cleanup() functionality implemented in subclasses. # By detecting if the current instance overrides cleanup(), we can determine # if the deprecated path of calling cleanup() should be performed - which avoids # unnecessary deprecation warnings in a majority of configurations in which # subclassed KernelManager instances are not in use. # Note: because subclasses may have already implemented cleanup_resources() # but need to support older jupyter_clients, we should only take the deprecated # path if cleanup() is overridden but cleanup_resources() is not. overrides_cleanup = type(self).cleanup is not KernelManager.cleanup overrides_cleanup_resources = ( type(self).cleanup_resources is not KernelManager.cleanup_resources ) if overrides_cleanup and not overrides_cleanup_resources: self.cleanup(connection_file=not restart) else: self.cleanup_resources(restart=restart) shutdown_kernel = run_sync(_async_shutdown_kernel) async def _async_restart_kernel(self, now: bool = False, newports: bool = False, **kw) -> None: """Restarts a kernel with the arguments that were used to launch it. Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. newports : bool, optional If the old kernel was launched with random ports, this flag decides whether the same ports and connection file will be used again. If False, the same ports and connection file are used. This is the default. If True, new random port numbers are chosen and a new connection file is written. It is still possible that the newly chosen random port numbers happen to be the same as the old ones. `**kw` : optional Any options specified here will overwrite those used to launch the kernel. """ if self._launch_args is None: raise RuntimeError("Cannot restart the kernel. " "No previous call to 'start_kernel'.") else: # Stop currently running kernel. await ensure_async(self.shutdown_kernel(now=now, restart=True)) if newports: self.cleanup_random_ports() # Start new kernel. self._launch_args.update(kw) await ensure_async(self.start_kernel(**self._launch_args)) restart_kernel = run_sync(_async_restart_kernel) @property def has_kernel(self) -> bool: """Has a kernel been started that we are managing.""" return self.kernel is not None async def _async_send_kernel_sigterm(self) -> None: """similar to _kill_kernel, but with sigterm (not sigkill), but do not block""" if self.has_kernel: # Signal the kernel to terminate (sends SIGTERM on Unix and # if the kernel is a subprocess and we are on windows; this is # equivalent to kill try: if hasattr(self.kernel, "terminate"): self.kernel.terminate() elif hasattr(signal, "SIGTERM"): await self._async_signal_kernel(signal.SIGTERM) else: self.log.debug( "Cannot set term signal to kernel, no" " `.terminate()` method and no values for SIGTERM" ) except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. if sys.platform == "win32": if e.winerror != 5: # type: ignore raise # On Unix, we may get an ESRCH error if the process has already # terminated. Ignore it. else: from errno import ESRCH if e.errno != ESRCH: raise _send_kernel_sigterm = run_sync(_async_send_kernel_sigterm) async def _async_kill_kernel(self) -> None: """Kill the running kernel. This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: if hasattr(signal, "SIGKILL"): await self._async_signal_kernel(signal.SIGKILL) # type: ignore else: self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. if sys.platform == "win32": if e.winerror != 5: # type: ignore raise # On Unix, we may get an ESRCH error if the process has already # terminated. Ignore it. else: from errno import ESRCH if e.errno != ESRCH: raise # Wait until the kernel terminates. try: await asyncio.wait_for(self._async_wait(), timeout=5.0) except asyncio.TimeoutError: # Wait timed out, just log warning but continue - not much more we can do. self.log.warning("Wait for final termination of kernel timed out - continuing...") pass else: # Process is no longer alive, wait and clear if self.kernel is not None: while self.kernel.poll() is None: await asyncio.sleep(0.1) self.kernel = None _kill_kernel = run_sync(_async_kill_kernel) async def _async_interrupt_kernel(self) -> None: """Interrupts the kernel by sending it a signal. Unlike ``signal_kernel``, this operation is well supported on all platforms. """ if self.has_kernel: assert self.kernel_spec is not None interrupt_mode = self.kernel_spec.interrupt_mode if interrupt_mode == "signal": if sys.platform == "win32": from .win_interrupt import send_interrupt send_interrupt(self.kernel.win32_interrupt_event) else: await self._async_signal_kernel(signal.SIGINT) elif interrupt_mode == "message": msg = self.session.msg("interrupt_request", content={}) self._connect_control_socket() self.session.send(self._control_socket, msg) else: raise RuntimeError("Cannot interrupt kernel. No kernel is running!") interrupt_kernel = run_sync(_async_interrupt_kernel) async def _async_signal_kernel(self, signum: int) -> None: """Sends a signal to the process group of the kernel (this usually includes the kernel and any subprocesses spawned by the kernel). Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ if self.has_kernel: if hasattr(os, "getpgid") and hasattr(os, "killpg"): try: pgid = os.getpgid(self.kernel.pid) # type: ignore os.killpg(pgid, signum) # type: ignore return except OSError: pass self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") signal_kernel = run_sync(_async_signal_kernel) async def _async_is_alive(self) -> bool: """Is the kernel process still running?""" if self.has_kernel: if self.kernel.poll() is None: return True else: return False else: # we don't have a kernel return False is_alive = run_sync(_async_is_alive) async def _async_wait(self, pollinterval: float = 0.1) -> None: # Use busy loop at 100ms intervals, polling until the process is # not alive. If we find the process is no longer alive, complete # its cleanup via the blocking wait(). Callers are responsible for # issuing calls to wait() using a timeout (see _kill_kernel()). while await self._async_is_alive(): await asyncio.sleep(pollinterval)
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp, ConnectionFileMixin): name = 'ipython-kernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session] # the kernel class, as an importstring kernel_class = Type('ipykernel.ipkernel.IPythonKernel', klass='ipykernel.kernelbase.Kernel', help="""The Kernel subclass to be used. This should allow easy re-use of the IPKernelApp entry point to configure and launch kernels other than IPython's own. """).tag(config=True) kernel = Any() poller = Any( ) # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat, allow_none=True) ports = Dict() subcommands = { 'install': ('ipykernel.kernelspec.InstallIPythonKernelSpecApp', 'Install the IPython kernel'), } # connection info: connection_dir = Unicode() @default('connection_dir') def _default_connection_dir(self): return jupyter_runtime_dir() @property def abs_connection_file(self): if os.path.basename(self.connection_file) == self.connection_file: return os.path.join(self.connection_dir, self.connection_file) else: return self.connection_file # streams, etc. no_stdout = Bool( False, help="redirect stdout to the null device").tag(config=True) no_stderr = Bool( False, help="redirect stderr to the null device").tag(config=True) outstream_class = DottedObjectName( 'ipykernel.iostream.OutStream', help="The importstring for the OutStream factory").tag(config=True) displayhook_class = DottedObjectName( 'ipykernel.displayhook.ZMQDisplayHook', help="The importstring for the DisplayHook factory").tag(config=True) # polling parent_handle = Integer( int(os.environ.get('JPY_PARENT_PID') or 0), help="""kill this process if its parent dies. On Windows, the argument specifies the HANDLE of the parent process, otherwise it is simply boolean. """).tag(config=True) interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0), help="""ONLY USED ON WINDOWS Interrupt this process when the parent is signaled. """).tag(config=True) def init_crash_handler(self): sys.excepthook = self.excepthook def excepthook(self, etype, evalue, tb): # write uncaught traceback to 'real' stderr, not zmq-forwarder traceback.print_exception(etype, evalue, tb, file=sys.__stderr__) def init_poller(self): if sys.platform == 'win32': if self.interrupt or self.parent_handle: self.poller = ParentPollerWindows(self.interrupt, self.parent_handle) elif self.parent_handle and self.parent_handle != 1: # PID 1 (init) is special and will never go away, # only be reassigned. # Parent polling doesn't work if ppid == 1 to start with. self.poller = ParentPollerUnix() def _bind_socket(self, s, port): iface = '%s://%s' % (self.transport, self.ip) if self.transport == 'tcp': if port <= 0: port = s.bind_to_random_port(iface) else: s.bind("tcp://%s:%i" % (self.ip, port)) elif self.transport == 'ipc': if port <= 0: port = 1 path = "%s-%i" % (self.ip, port) while os.path.exists(path): port = port + 1 path = "%s-%i" % (self.ip, port) else: path = "%s-%i" % (self.ip, port) s.bind("ipc://%s" % path) return port def write_connection_file(self): """write connection info to JSON file""" cf = self.abs_connection_file self.log.debug("Writing connection file: %s", cf) write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port, control_port=self.control_port) def cleanup_connection_file(self): cf = self.abs_connection_file self.log.debug("Cleaning up connection file: %s", cf) try: os.remove(cf) except (IOError, OSError): pass self.cleanup_ipc_files() def init_connection_file(self): if not self.connection_file: self.connection_file = "kernel-%s.json" % os.getpid() try: self.connection_file = filefind(self.connection_file, ['.', self.connection_dir]) except IOError: self.log.debug("Connection file not found: %s", self.connection_file) # This means I own it, and I'll create it in this directory: ensure_dir_exists(os.path.dirname(self.abs_connection_file), 0o700) # Also, I will clean it up: atexit.register(self.cleanup_connection_file) return try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_sockets(self): # Create a context, a session, and the kernel sockets. self.log.info("Starting the kernel at pid: %i", os.getpid()) context = zmq.Context.instance() # Uncomment this to try closing the context. # atexit.register(context.term) self.shell_socket = context.socket(zmq.ROUTER) self.shell_socket.linger = 1000 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_socket.linger = 1000 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) self.control_socket = context.socket(zmq.ROUTER) self.control_socket.linger = 1000 self.control_port = self._bind_socket(self.control_socket, self.control_port) self.log.debug("control ROUTER Channel on port: %i" % self.control_port) self.init_iopub(context) def init_iopub(self, context): self.iopub_socket = context.socket(zmq.PUB) self.iopub_socket.linger = 1000 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) self.configure_tornado_logger() self.iopub_thread = IOPubThread(self.iopub_socket, pipe=True) self.iopub_thread.start() # backward-compat: wrap iopub socket API in background thread self.iopub_socket = self.iopub_thread.background_socket def init_heartbeat(self): """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) self.heartbeat.start() def log_connection_info(self): """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.connection_dir: # use shortname tail = basename else: tail = self.connection_file lines = [ "To connect another client to this kernel, use:", " --existing %s" % tail, ] # log connection info # info-level, so often not shown. # frontends should use the %connect_info magic # to see the connection info for line in lines: self.log.info(line) # also raw print to the terminal if no parent_handle (`ipython kernel`) # unless log-level is CRITICAL (--quiet) if not self.parent_handle and self.log_level < logging.CRITICAL: io.rprint(_ctrl_c_message) for line in lines: io.rprint(line) self.ports = dict(shell=self.shell_port, iopub=self.iopub_port, stdin=self.stdin_port, hb=self.hb_port, control=self.control_port) def init_blackhole(self): """redirects stdout/stderr to devnull if necessary""" if self.no_stdout or self.no_stderr: blackhole = open(os.devnull, 'w') if self.no_stdout: sys.stdout = sys.__stdout__ = blackhole if self.no_stderr: sys.stderr = sys.__stderr__ = blackhole def init_io(self): """Redirect input streams and set a display hook.""" if self.outstream_class: outstream_factory = import_item(str(self.outstream_class)) sys.stdout = outstream_factory(self.session, self.iopub_thread, u'stdout') sys.stderr = outstream_factory(self.session, self.iopub_thread, u'stderr') if self.displayhook_class: displayhook_factory = import_item(str(self.displayhook_class)) self.displayhook = displayhook_factory(self.session, self.iopub_socket) sys.displayhook = self.displayhook self.patch_io() def patch_io(self): """Patch important libraries that can't handle sys.stdout forwarding""" try: import faulthandler except ImportError: pass else: # Warning: this is a monkeypatch of `faulthandler.enable`, watch for possible # updates to the upstream API and update accordingly (up-to-date as of Python 3.5): # https://docs.python.org/3/library/faulthandler.html#faulthandler.enable # change default file to __stderr__ from forwarded stderr faulthandler_enable = faulthandler.enable def enable(file=sys.__stderr__, all_threads=True, **kwargs): return faulthandler_enable(file=file, all_threads=all_threads, **kwargs) faulthandler.enable = enable if hasattr(faulthandler, 'register'): faulthandler_register = faulthandler.register def register(signum, file=sys.__stderr__, all_threads=True, chain=False, **kwargs): return faulthandler_register(signum, file=file, all_threads=all_threads, chain=chain, **kwargs) faulthandler.register = register def init_signal(self): signal.signal(signal.SIGINT, signal.SIG_IGN) def init_kernel(self): """Create the Kernel object itself""" shell_stream = ZMQStream(self.shell_socket) control_stream = ZMQStream(self.control_socket) kernel_factory = self.kernel_class.instance kernel = kernel_factory( parent=self, session=self.session, shell_streams=[shell_stream, control_stream], iopub_thread=self.iopub_thread, iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, user_ns=self.user_ns, ) kernel.record_ports( {name + '_port': port for name, port in self.ports.items()}) self.kernel = kernel # Allow the displayhook to get the execution count self.displayhook.get_execution_count = lambda: kernel.execution_count def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" # Register inline backend as default # this is higher priority than matplotlibrc, # but lower priority than anything else (mpl.use() for instance). # This only affects matplotlib >= 1.5 if not os.environ.get('MPLBACKEND'): os.environ[ 'MPLBACKEND'] = 'module://ipykernel.pylab.backend_inline' # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` # to ensure that any exception is printed straight to stderr. # Normally _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. shell = self.shell _showtraceback = shell._showtraceback try: # replace error-sending traceback with stderr def print_tb(etype, evalue, stb): print("GUI event loop or pylab initialization failed", file=sys.stderr) print(shell.InteractiveTB.stb2text(stb), file=sys.stderr) shell._showtraceback = print_tb InteractiveShellApp.init_gui_pylab(self) finally: shell._showtraceback = _showtraceback def init_shell(self): self.shell = getattr(self.kernel, 'shell', None) if self.shell: self.shell.configurables.append(self) def init_extensions(self): super(IPKernelApp, self).init_extensions() # BEGIN HARDCODED WIDGETS HACK # Ensure ipywidgets extension is loaded if available extension_man = self.shell.extension_manager if 'ipywidgets' not in extension_man.loaded: try: extension_man.load_extension('ipywidgets') except ImportError as e: self.log.debug( 'ipywidgets package not installed. Widgets will not be available.' ) # END HARDCODED WIDGETS HACK def configure_tornado_logger(self): """ Configure the tornado logging.Logger. Must set up the tornado logger or else tornado will call basicConfig for the root logger which makes the root logger go to the real sys.stderr instead of the capture streams. This function mimics the setup of logging.basicConfig. """ logger = logging.getLogger('tornado') handler = logging.StreamHandler() formatter = logging.Formatter(logging.BASIC_FORMAT) handler.setFormatter(formatter) logger.addHandler(handler) @catch_config_error def initialize(self, argv=None): super(IPKernelApp, self).initialize(argv) if self.subapp is not None: return # register zmq IOLoop with tornado zmq_ioloop.install() self.init_blackhole() self.init_connection_file() self.init_poller() self.init_sockets() self.init_heartbeat() # writing/displaying connection info must be *after* init_sockets/heartbeat self.write_connection_file() # Log connection info after writing connection file, so that the connection # file is definitely available at the time someone reads the log. self.log_connection_info() self.init_io() self.init_signal() self.init_kernel() # shell init steps self.init_path() self.init_shell() if self.shell: self.init_gui_pylab() self.init_extensions() self.init_code() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request sys.stdout.flush() sys.stderr.flush() def start(self): if self.subapp is not None: return self.subapp.start() if self.poller is not None: self.poller.start() self.kernel.start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: pass
class KernelManager(ConnectionFileMixin): """Manages a single kernel in a subprocess on this host. This version starts kernels with Popen. """ # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.Context) def _context_default(self): return zmq.Context.instance() # the class to create with our `client` method client_class = DottedObjectName('jupyter_client.blocking.BlockingKernelClient') client_factory = Type(klass='jupyter_client.KernelClient') def _client_factory_default(self): return import_item(self.client_class) def _client_class_changed(self, name, old, new): self.client_factory = import_item(str(new)) # The kernel process with which the KernelManager is communicating. # generally a Popen instance kernel = Any() kernel_spec_manager = Instance(kernelspec.KernelSpecManager) def _kernel_spec_manager_default(self): return kernelspec.KernelSpecManager(data_dir=self.data_dir) def _kernel_spec_manager_changed(self): self._kernel_spec = None kernel_name = Unicode(kernelspec.NATIVE_KERNEL_NAME) def _kernel_name_changed(self, name, old, new): self._kernel_spec = None if new == 'python': self.kernel_name = kernelspec.NATIVE_KERNEL_NAME _kernel_spec = None @property def kernel_spec(self): if self._kernel_spec is None: self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec kernel_cmd = List(Unicode(), config=True, help="""DEPRECATED: Use kernel_name instead. The Popen Command to launch the kernel. Override this if you have a custom kernel. If kernel_cmd is specified in a configuration file, Jupyter does not pass any arguments to the kernel, because it cannot make any assumptions about the arguments that the kernel understands. In particular, this means that the kernel does not receive the option --debug if it given on the Jupyter command line. """ ) def _kernel_cmd_changed(self, name, old, new): warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to " "start different kernels.") @property def ipykernel(self): return self.kernel_name in {'python', 'python2', 'python3'} # Protected traits _launch_args = Any() _control_socket = Any() _restarter = Any() autorestart = Bool(False, config=True, help="""Should we autorestart the kernel if it dies.""" ) def __del__(self): self._close_control_socket() self.cleanup_connection_file() #-------------------------------------------------------------------------- # Kernel restarter #-------------------------------------------------------------------------- def start_restarter(self): pass def stop_restarter(self): pass def add_restart_callback(self, callback, event='restart'): """register a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.add_callback(callback, event) def remove_restart_callback(self, callback, event='restart'): """unregister a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.remove_callback(callback, event) #-------------------------------------------------------------------------- # create a Client connected to our Kernel #-------------------------------------------------------------------------- def client(self, **kwargs): """Create a client configured to connect to our kernel""" kw = {} kw.update(self.get_connection_info()) kw.update(dict( connection_file=self.connection_file, session=self.session, parent=self, )) # add kwargs last, for manual overrides kw.update(kwargs) return self.client_factory(**kw) #-------------------------------------------------------------------------- # Kernel management #-------------------------------------------------------------------------- def format_kernel_cmd(self, extra_arguments=None): """replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] if self.kernel_cmd: cmd = self.kernel_cmd + extra_arguments else: cmd = self.kernel_spec.argv + extra_arguments ns = dict(connection_file=self.connection_file, prefix=sys.prefix, ) ns.update(self._launch_args) pat = re.compile(r'\{([A-Za-z0-9_]+)\}') def from_ns(match): """Get the key out of ns if it's there, otherwise no change.""" return ns.get(match.group(1), match.group()) return [ pat.sub(from_ns, arg) for arg in cmd ] def _launch_kernel(self, kernel_cmd, **kw): """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ return launch_kernel(kernel_cmd, **kw) # Control socket used for polite kernel shutdown def _connect_control_socket(self): if self._control_socket is None: self._control_socket = self.connect_control() self._control_socket.linger = 100 def _close_control_socket(self): if self._control_socket is None: return self._control_socket.close() self._control_socket = None def start_kernel(self, **kw): """Starts a kernel on this host in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError("Can only launch a kernel on a local interface. " "Make sure that the '*_address' attributes are " "configured properly. " "Currently valid addresses are: %s" % local_ips() ) # write connection file / get default ports self.write_connection_file() # save kwargs for use in restart self._launch_args = kw.copy() # build the Popen cmd extra_arguments = kw.pop('extra_arguments', []) kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) env = os.environ.copy() # Don't allow PYTHONEXECUTABLE to be passed to kernel process. # If set, it can bork all the things. env.pop('PYTHONEXECUTABLE', None) if not self.kernel_cmd: # If kernel_cmd has been set manually, don't refer to a kernel spec # Environment variables from kernel spec are added to os.environ env.update(self.kernel_spec.env or {}) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) self.kernel = self._launch_kernel(kernel_cmd, env=env, **kw) self.start_restarter() self._connect_control_socket() def request_shutdown(self, restart=False): """Send a shutdown request via control channel """ content = dict(restart=restart) msg = self.session.msg("shutdown_request", content=content) self.session.send(self._control_socket, msg) def finish_shutdown(self, waittime=1, pollinterval=0.1): """Wait for kernel shutdown, then kill process if it doesn't shutdown. This does not send shutdown requests - use :meth:`request_shutdown` first. """ for i in range(int(waittime/pollinterval)): if self.is_alive(): time.sleep(pollinterval) else: break else: # OK, we've waited long enough. if self.has_kernel: self._kill_kernel() def cleanup(self, connection_file=True): """Clean up resources when the kernel is shut down""" if connection_file: self.cleanup_connection_file() self.cleanup_ipc_files() self._close_control_socket() def shutdown_kernel(self, now=False, restart=False): """Attempts to the stop the kernel process cleanly. This attempts to shutdown the kernels cleanly by: 1. Sending it a shutdown message over the shell channel. 2. If that fails, the kernel is shutdown forcibly by sending it a signal. Parameters ---------- now : bool Should the kernel be forcible killed *now*. This skips the first, nice shutdown attempt. restart: bool Will this kernel be restarted after it is shutdown. When this is True, connection files will not be cleaned up. """ # Stop monitoring for restarting while we shutdown. self.stop_restarter() if now: self._kill_kernel() else: self.request_shutdown(restart=restart) # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. self.finish_shutdown() self.cleanup(connection_file=not restart) def restart_kernel(self, now=False, **kw): """Restarts a kernel with the arguments that were used to launch it. If the old kernel was launched with random ports, the same ports will be used for the new kernel. The same connection file is used again. Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. `**kw` : optional Any options specified here will overwrite those used to launch the kernel. """ if self._launch_args is None: raise RuntimeError("Cannot restart the kernel. " "No previous call to 'start_kernel'.") else: # Stop currently running kernel. self.shutdown_kernel(now=now, restart=True) # Start new kernel. self._launch_args.update(kw) self.start_kernel(**self._launch_args) @property def has_kernel(self): """Has a kernel been started that we are managing.""" return self.kernel is not None def _kill_kernel(self): """Kill the running kernel. This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. if sys.platform == 'win32': if e.winerror != 5: raise # On Unix, we may get an ESRCH error if the process has already # terminated. Ignore it. else: from errno import ESRCH if e.errno != ESRCH: raise # Block until the kernel terminates. self.kernel.wait() self.kernel = None else: raise RuntimeError("Cannot kill kernel. No kernel is running!") def interrupt_kernel(self): """Interrupts the kernel by sending it a signal. Unlike ``signal_kernel``, this operation is well supported on all platforms. """ if self.has_kernel: if sys.platform == 'win32': from .win_interrupt import send_interrupt send_interrupt(self.kernel.win32_interrupt_event) else: self.signal_kernel(signal.SIGINT) else: raise RuntimeError("Cannot interrupt kernel. No kernel is running!") def signal_kernel(self, signum): """Sends a signal to the process group of the kernel (this usually includes the kernel and any subprocesses spawned by the kernel). Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ if self.has_kernel: if hasattr(os, "getpgid") and hasattr(os, "killpg"): try: pgid = os.getpgid(self.kernel.pid) os.killpg(pgid, signum) return except OSError: pass self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") def is_alive(self): """Is the kernel process still running?""" if self.has_kernel: if self.kernel.poll() is None: return True else: return False else: # we don't have a kernel return False
class AsyncKernelManager(KernelManager): """Manages kernels in an asynchronous manner """ client_class = DottedObjectName( 'jupyter_client.asynchronous.AsyncKernelClient') client_factory = Type( klass='jupyter_client.asynchronous.AsyncKernelClient') async def _launch_kernel(self, kernel_cmd, **kw): """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ res = launch_kernel(kernel_cmd, **kw) return res async def start_kernel(self, **kw): """Starts a kernel in a separate process in an asynchronous manner. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ kernel_cmd, kw = self.pre_start_kernel(**kw) # launch the kernel subprocess self.log.debug("Starting kernel (async): %s", kernel_cmd) self.kernel = await self._launch_kernel(kernel_cmd, **kw) self.post_start_kernel(**kw) async def finish_shutdown(self, waittime=None, pollinterval=0.1): """Wait for kernel shutdown, then kill process if it doesn't shutdown. This does not send shutdown requests - use :meth:`request_shutdown` first. """ if waittime is None: waittime = max(self.shutdown_wait_time, 0) try: await asyncio.wait_for(self._async_wait(pollinterval=pollinterval), timeout=waittime) except asyncio.TimeoutError: self.log.debug("Kernel is taking too long to finish, killing") await self._kill_kernel() else: # Process is no longer alive, wait and clear if self.kernel is not None: self.kernel.wait() self.kernel = None async def shutdown_kernel(self, now=False, restart=False): """Attempts to stop the kernel process cleanly. This attempts to shutdown the kernels cleanly by: 1. Sending it a shutdown message over the shell channel. 2. If that fails, the kernel is shutdown forcibly by sending it a signal. Parameters ---------- now : bool Should the kernel be forcible killed *now*. This skips the first, nice shutdown attempt. restart: bool Will this kernel be restarted after it is shutdown. When this is True, connection files will not be cleaned up. """ # Stop monitoring for restarting while we shutdown. self.stop_restarter() if now: await self._kill_kernel() else: self.request_shutdown(restart=restart) # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. await self.finish_shutdown() # See comment in KernelManager.shutdown_kernel(). overrides_cleanup = type( self).cleanup is not AsyncKernelManager.cleanup overrides_cleanup_resources = type( self).cleanup_resources is not AsyncKernelManager.cleanup_resources if overrides_cleanup and not overrides_cleanup_resources: self.cleanup(connection_file=not restart) else: self.cleanup_resources(restart=restart) async def restart_kernel(self, now=False, newports=False, **kw): """Restarts a kernel with the arguments that were used to launch it. Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. newports : bool, optional If the old kernel was launched with random ports, this flag decides whether the same ports and connection file will be used again. If False, the same ports and connection file are used. This is the default. If True, new random port numbers are chosen and a new connection file is written. It is still possible that the newly chosen random port numbers happen to be the same as the old ones. `**kw` : optional Any options specified here will overwrite those used to launch the kernel. """ if self._launch_args is None: raise RuntimeError("Cannot restart the kernel. " "No previous call to 'start_kernel'.") else: # Stop currently running kernel. await self.shutdown_kernel(now=now, restart=True) if newports: self.cleanup_random_ports() # Start new kernel. self._launch_args.update(kw) await self.start_kernel(**self._launch_args) return None async def _kill_kernel(self): """Kill the running kernel. This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: if hasattr(signal, 'SIGKILL'): await self.signal_kernel(signal.SIGKILL) else: self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. if sys.platform == 'win32': if e.winerror != 5: raise # On Unix, we may get an ESRCH error if the process has already # terminated. Ignore it. else: from errno import ESRCH if e.errno != ESRCH: raise # Wait until the kernel terminates. try: await asyncio.wait_for(self._async_wait(), timeout=5.0) except asyncio.TimeoutError: # Wait timed out, just log warning but continue - not much more we can do. self.log.warning( "Wait for final termination of kernel timed out - continuing..." ) pass else: # Process is no longer alive, wait and clear if self.kernel is not None: self.kernel.wait() self.kernel = None async def interrupt_kernel(self): """Interrupts the kernel by sending it a signal. Unlike ``signal_kernel``, this operation is well supported on all platforms. """ if self.has_kernel: interrupt_mode = self.kernel_spec.interrupt_mode if interrupt_mode == 'signal': if sys.platform == 'win32': from .win_interrupt import send_interrupt send_interrupt(self.kernel.win32_interrupt_event) else: await self.signal_kernel(signal.SIGINT) elif interrupt_mode == 'message': msg = self.session.msg("interrupt_request", content={}) self._connect_control_socket() self.session.send(self._control_socket, msg) else: raise RuntimeError( "Cannot interrupt kernel. No kernel is running!") async def signal_kernel(self, signum): """Sends a signal to the process group of the kernel (this usually includes the kernel and any subprocesses spawned by the kernel). Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ if self.has_kernel: if hasattr(os, "getpgid") and hasattr(os, "killpg"): try: pgid = os.getpgid(self.kernel.pid) os.killpg(pgid, signum) return except OSError: pass self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") async def is_alive(self): """Is the kernel process still running?""" if self.has_kernel: if self.kernel.poll() is None: return True else: return False else: # we don't have a kernel return False async def _async_wait(self, pollinterval=0.1): # Use busy loop at 100ms intervals, polling until the process is # not alive. If we find the process is no longer alive, complete # its cleanup via the blocking wait(). Callers are responsible for # issuing calls to wait() using a timeout (see _kill_kernel()). while await self.is_alive(): await asyncio.sleep(pollinterval)
class IPClusterEngines(BaseParallelApplication): name = u'ipcluster' description = engines_help examples = _engines_examples usage = None default_log_level = logging.INFO classes = List() def _classes_default(self): from ipyparallel.apps import launcher launchers = launcher.all_launchers eslaunchers = [l for l in launchers if 'EngineSet' in l.__name__] return [ProfileDir] + eslaunchers n = Integer( num_cpus(), config=True, help= """The number of engines to start. The default is to use one for each CPU on your machine""") engine_launcher = Any(config=True, help="Deprecated, use engine_launcher_class") @observe('engine_launcher') def _engine_launcher_changed(self, change): if isinstance(change['new'], string_types): self.log.warn( "WARNING: %s.engine_launcher is deprecated as of 0.12," " use engine_launcher_class" % self.__class__.__name__) self.engine_launcher_class = change['new'] engine_launcher_class = DottedObjectName( 'LocalEngineSetLauncher', config=True, help="""The class for launching a set of Engines. Change this value to use various batch systems to launch your engines, such as PBS,SGE,MPI,etc. Each launcher class has its own set of configuration options, for making sure it will work in your environment. You can also write your own launcher, and specify it's absolute import path, as in 'mymodule.launcher.FTLEnginesLauncher`. IPython's bundled examples include: Local : start engines locally as subprocesses [default] MPI : use mpiexec to launch engines in an MPI environment PBS : use PBS (qsub) to submit engines to a batch queue SGE : use SGE (qsub) to submit engines to a batch queue LSF : use LSF (bsub) to submit engines to a batch queue SSH : use SSH to start the controller Note that SSH does *not* move the connection files around, so you will likely have to do this manually unless the machines are on a shared file system. HTCondor : use HTCondor to submit engines to a batch queue Slurm : use Slurm to submit engines to a batch queue WindowsHPC : use Windows HPC If you are using one of IPython's builtin launchers, you can specify just the prefix, e.g: c.IPClusterEngines.engine_launcher_class = 'SSH' or: ipcluster start --engines=MPI """) daemonize = Bool( False, config=True, help="""Daemonize the ipcluster program. This implies --log-to-file. Not available on Windows. """) @observe('daemonize') def _daemonize_changed(self, change): if change['new']: self.log_to_file = True early_shutdown = Integer(30, config=True, help="The timeout (in seconds)") _stopping = False aliases = Dict(engine_aliases) flags = Dict(engine_flags) @catch_config_error def initialize(self, argv=None): super(IPClusterEngines, self).initialize(argv) self.init_signal() self.init_launchers() def init_launchers(self): self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet') def init_signal(self): # Setup signals signal.signal(signal.SIGINT, self.sigint_handler) def build_launcher(self, clsname, kind=None): """import and instantiate a Launcher based on importstring""" try: klass = find_launcher_class(clsname, kind) except (ImportError, KeyError): self.log.fatal("Could not import launcher class: %r" % clsname) self.exit(1) launcher = klass( work_dir=u'.', parent=self, log=self.log, profile_dir=self.profile_dir.location, cluster_id=self.cluster_id, ) return launcher def engines_started_ok(self): if self.engine_launcher.running: self.log.info("Engines appear to have started successfully") self.early_shutdown = 0 def start_engines(self): # Some EngineSetLaunchers ignore `n` and use their own engine count, such as SSH: n = getattr(self.engine_launcher, 'engine_count', self.n) self.log.info("Starting %s Engines with %s", n, self.engine_launcher_class) try: self.engine_launcher.start(self.n) except: self.log.exception("Engine start failed") raise self.engine_launcher.on_stop(self.engines_stopped_early) if self.early_shutdown: self.loop.add_timeout(self.loop.time() + self.early_shutdown, self.engines_started_ok) def engines_stopped_early(self, r): if self.early_shutdown and not self._stopping: self.log.error(""" Engines shutdown early, they probably failed to connect. Check the engine log files for output. If your controller and engines are not on the same machine, you probably have to instruct the controller to listen on an interface other than localhost. You can set this by adding "--ip='*'" to your ControllerLauncher.controller_args. Be sure to read our security docs before instructing your controller to listen on a public interface. """) self.stop_launchers() return self.engines_stopped(r) def engines_stopped(self, r): return self.loop.stop() def stop_engines(self): if self.engine_launcher.running: self.log.info("Stopping Engines...") d = self.engine_launcher.stop() return d else: return None def stop_launchers(self, r=None): if not self._stopping: self._stopping = True self.log.error("IPython cluster: stopping") self.stop_engines() # Wait a few seconds to let things shut down. self.loop.add_timeout(self.loop.time() + 3, self.loop.stop) def sigint_handler(self, signum, frame): self.log.debug("SIGINT received, stopping launchers...") self.stop_launchers() def start_logging(self): # Remove old log files of the controller and engine if self.clean_logs: log_dir = self.profile_dir.log_dir for f in os.listdir(log_dir): if re.match(r'ip(engine|controller)-.+\.(log|err|out)', f): os.remove(os.path.join(log_dir, f)) def start(self): """Start the app for the engines subcommand.""" self.log.info("IPython cluster: started") # First see if the cluster is already running # Now log and daemonize self.log.info('Starting engines with [daemon=%r]' % self.daemonize) # TODO: Get daemonize working on Windows or as a Windows Server. if self.daemonize: if os.name == 'posix': daemonize() self.loop.add_callback(self.start_engines) # Now write the new pid file AFTER our new forked pid is active. # self.write_pid_file() try: self.loop.start() except KeyboardInterrupt: pass except zmq.ZMQError as e: if e.errno == errno.EINTR: pass else: raise
class IPClusterStart(IPClusterEngines): name = u'ipcluster' description = start_help examples = _start_examples default_log_level = logging.INFO auto_create = Bool( True, config=True, help="whether to create the profile_dir if it doesn't exist") classes = List() def _classes_default(self, ): from ipyparallel.apps import launcher return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers clean_logs = Bool(True, config=True, help="whether to cleanup old logs before starting") delay = CFloat( 1., config=True, help="delay (in s) between starting the controller and the engines") controller_ip = Unicode(config=True, help="Set the IP address of the controller.") controller_location = Unicode( config=True, help="""Set the location (hostname or ip) of the controller. This is used by engines and clients to locate the controller when the controller listens on all interfaces """) controller_launcher = Any(config=True, help="Deprecated, use controller_launcher_class") @observe('controller_launcher') def _controller_launcher_changed(self, change): if isinstance(change['new'], string_types): # old 0.11-style config self.log.warn( "WARNING: %s.controller_launcher is deprecated as of 0.12," " use controller_launcher_class" % self.__class__.__name__) self.controller_launcher_class = change['new'] controller_launcher_class = DottedObjectName( 'LocalControllerLauncher', config=True, help= """The class for launching a Controller. Change this value if you want your controller to also be launched by a batch system, such as PBS,SGE,MPI,etc. Each launcher class has its own set of configuration options, for making sure it will work in your environment. Note that using a batch launcher for the controller *does not* put it in the same batch job as the engines, so they will still start separately. IPython's bundled examples include: Local : start engines locally as subprocesses MPI : use mpiexec to launch the controller in an MPI universe PBS : use PBS (qsub) to submit the controller to a batch queue SGE : use SGE (qsub) to submit the controller to a batch queue LSF : use LSF (bsub) to submit the controller to a batch queue HTCondor : use HTCondor to submit the controller to a batch queue Slurm : use Slurm to submit engines to a batch queue SSH : use SSH to start the controller WindowsHPC : use Windows HPC If you are using one of IPython's builtin launchers, you can specify just the prefix, e.g: c.IPClusterStart.controller_launcher_class = 'SSH' or: ipcluster start --controller=MPI """) reset = Bool(False, config=True, help="Whether to reset config files as part of '--create'.") # flags = Dict(flags) aliases = Dict(start_aliases) def init_launchers(self): self.controller_launcher = self.build_launcher( self.controller_launcher_class, 'Controller') controller_args = getattr(self.controller_launcher, 'controller_args', None) if controller_args is None: def add_args(args): # only some Launchers support modifying controller args self.log.warning( "Not adding controller args %s. " "controller_args passthrough is not supported by %s", args, self.controller_launcher.__class__.__name__, ) else: add_args = controller_args.extend if self.controller_ip: add_args(['--ip=%s' % self.controller_ip]) if self.controller_location: add_args(['--location=%s' % self.controller_location]) if self.extra_args: add_args(self.extra_args) self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet') def engines_stopped(self, r): """prevent parent.engines_stopped from stopping everything on engine shutdown""" pass def start_controller(self): self.log.info("Starting Controller with %s", self.controller_launcher_class) self.controller_launcher.on_stop(self.stop_launchers) try: self.controller_launcher.start() except: self.log.exception("Controller start failed") raise def stop_controller(self): # self.log.info("In stop_controller") if self.controller_launcher and self.controller_launcher.running: return self.controller_launcher.stop() def stop_launchers(self, r=None): if not self._stopping: self.stop_controller() super(IPClusterStart, self).stop_launchers() def start(self): """Start the app for the start subcommand.""" # First see if the cluster is already running try: pid = self.get_pid_from_file() except PIDFileError: pass else: if self.check_pid(pid): self.log.critical('Cluster is already running with [pid=%s]. ' 'use "ipcluster stop" to stop the cluster.' % pid) # Here I exit with a unusual exit status that other processes # can watch for to learn how I existed. self.exit(ALREADY_STARTED) else: self.remove_pid_file() # Now log and daemonize self.log.info('Starting ipcluster with [daemon=%r]' % self.daemonize) # TODO: Get daemonize working on Windows or as a Windows Server. if self.daemonize: if os.name == 'posix': daemonize() def start(): self.start_controller() self.loop.add_timeout(self.loop.time() + self.delay, self.start_engines) self.loop.add_callback(start) # Now write the new pid file AFTER our new forked pid is active. self.write_pid_file() try: self.loop.start() except KeyboardInterrupt: pass except zmq.ZMQError as e: if e.errno == errno.EINTR: pass else: raise finally: self.remove_pid_file()
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp, ConnectionFileMixin): name = 'ipython-kernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session] # the kernel class, as an importstring kernel_class = Type('ipykernel.ipkernel.IPythonKernel', config=True, klass='ipykernel.kernelbase.Kernel', help="""The Kernel subclass to be used. This should allow easy re-use of the IPKernelApp entry point to configure and launch kernels other than IPython's own. """) kernel = Any() poller = Any( ) # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat, allow_none=True) ports = Dict() # connection info: connection_dir = Unicode() def _connection_dir_default(self): d = jupyter_runtime_dir() ensure_dir_exists(d, 0o700) return d @property def abs_connection_file(self): if os.path.basename(self.connection_file) == self.connection_file: return os.path.join(self.connection_dir, self.connection_file) else: return self.connection_file # streams, etc. no_stdout = Bool(False, config=True, help="redirect stdout to the null device") no_stderr = Bool(False, config=True, help="redirect stderr to the null device") outstream_class = DottedObjectName( 'ipykernel.iostream.OutStream', config=True, help="The importstring for the OutStream factory") displayhook_class = DottedObjectName( 'ipykernel.displayhook.ZMQDisplayHook', config=True, help="The importstring for the DisplayHook factory") # polling parent_handle = Integer( int(os.environ.get('JPY_PARENT_PID') or 0), config=True, help="""kill this process if its parent dies. On Windows, the argument specifies the HANDLE of the parent process, otherwise it is simply boolean. """) interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0), config=True, help="""ONLY USED ON WINDOWS Interrupt this process when the parent is signaled. """) def init_crash_handler(self): # Install minimal exception handling sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor', ostream=sys.__stdout__) def init_poller(self): if sys.platform == 'win32': if self.interrupt or self.parent_handle: self.poller = ParentPollerWindows(self.interrupt, self.parent_handle) elif self.parent_handle: self.poller = ParentPollerUnix() def _bind_socket(self, s, port): iface = '%s://%s' % (self.transport, self.ip) if self.transport == 'tcp': if port <= 0: port = s.bind_to_random_port(iface) else: s.bind("tcp://%s:%i" % (self.ip, port)) elif self.transport == 'ipc': if port <= 0: port = 1 path = "%s-%i" % (self.ip, port) while os.path.exists(path): port = port + 1 path = "%s-%i" % (self.ip, port) else: path = "%s-%i" % (self.ip, port) s.bind("ipc://%s" % path) return port def write_connection_file(self): """write connection info to JSON file""" cf = self.abs_connection_file self.log.debug("Writing connection file: %s", cf) write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port, control_port=self.control_port) def cleanup_connection_file(self): cf = self.abs_connection_file self.log.debug("Cleaning up connection file: %s", cf) try: os.remove(cf) except (IOError, OSError): pass self.cleanup_ipc_files() def init_connection_file(self): if not self.connection_file: self.connection_file = "kernel-%s.json" % os.getpid() try: self.connection_file = filefind(self.connection_file, ['.', self.connection_dir]) except IOError: self.log.debug("Connection file not found: %s", self.connection_file) # This means I own it, so I will clean it up: atexit.register(self.cleanup_connection_file) return try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_sockets(self): # Create a context, a session, and the kernel sockets. self.log.info("Starting the kernel at pid: %i", os.getpid()) context = zmq.Context.instance() # Uncomment this to try closing the context. # atexit.register(context.term) self.shell_socket = context.socket(zmq.ROUTER) self.shell_socket.linger = 1000 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) self.iopub_socket = context.socket(zmq.PUB) self.iopub_socket.linger = 1000 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_socket.linger = 1000 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) self.control_socket = context.socket(zmq.ROUTER) self.control_socket.linger = 1000 self.control_port = self._bind_socket(self.control_socket, self.control_port) self.log.debug("control ROUTER Channel on port: %i" % self.control_port) def init_heartbeat(self): """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) self.heartbeat.start() def log_connection_info(self): """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.connection_dir: # use shortname tail = basename else: tail = self.connection_file lines = [ "To connect another client to this kernel, use:", " --existing %s" % tail, ] # log connection info # info-level, so often not shown. # frontends should use the %connect_info magic # to see the connection info for line in lines: self.log.info(line) # also raw print to the terminal if no parent_handle (`ipython kernel`) if not self.parent_handle: io.rprint(_ctrl_c_message) for line in lines: io.rprint(line) self.ports = dict(shell=self.shell_port, iopub=self.iopub_port, stdin=self.stdin_port, hb=self.hb_port, control=self.control_port) def init_blackhole(self): """redirects stdout/stderr to devnull if necessary""" if self.no_stdout or self.no_stderr: blackhole = open(os.devnull, 'w') if self.no_stdout: sys.stdout = sys.__stdout__ = blackhole if self.no_stderr: sys.stderr = sys.__stderr__ = blackhole def init_io(self): """Redirect input streams and set a display hook.""" if self.outstream_class: outstream_factory = import_item(str(self.outstream_class)) sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout') sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr') if self.displayhook_class: displayhook_factory = import_item(str(self.displayhook_class)) sys.displayhook = displayhook_factory(self.session, self.iopub_socket) def init_signal(self): signal.signal(signal.SIGINT, signal.SIG_IGN) def init_kernel(self): """Create the Kernel object itself""" shell_stream = ZMQStream(self.shell_socket) control_stream = ZMQStream(self.control_socket) kernel_factory = self.kernel_class.instance kernel = kernel_factory( parent=self, session=self.session, shell_streams=[shell_stream, control_stream], iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, user_ns=self.user_ns, ) kernel.record_ports(self.ports) self.kernel = kernel def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` # to ensure that any exception is printed straight to stderr. # Normally _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. shell = self.shell _showtraceback = shell._showtraceback try: # replace error-sending traceback with stderr def print_tb(etype, evalue, stb): print("GUI event loop or pylab initialization failed", file=io.stderr) print(shell.InteractiveTB.stb2text(stb), file=io.stderr) shell._showtraceback = print_tb InteractiveShellApp.init_gui_pylab(self) finally: shell._showtraceback = _showtraceback def init_shell(self): self.shell = getattr(self.kernel, 'shell', None) if self.shell: self.shell.configurables.append(self) def init_extensions(self): super(IPKernelApp, self).init_extensions() # BEGIN HARDCODED WIDGETS HACK # Ensure ipywidgets extension is loaded if available extension_man = self.shell.extension_manager if 'ipywidgets' not in extension_man.loaded: try: extension_man.load_extension('ipywidgets') except ImportError as e: self.log.debug( 'ipywidgets package not installed. Widgets will not be available.' ) # END HARDCODED WIDGETS HACK @catch_config_error def initialize(self, argv=None): super(IPKernelApp, self).initialize(argv) self.init_blackhole() self.init_connection_file() self.init_poller() self.init_sockets() self.init_heartbeat() # writing/displaying connection info must be *after* init_sockets/heartbeat self.log_connection_info() self.write_connection_file() self.init_io() self.init_signal() self.init_kernel() # shell init steps self.init_path() self.init_shell() if self.shell: self.init_gui_pylab() self.init_extensions() self.init_code() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request sys.stdout.flush() sys.stderr.flush() def start(self): if self.poller is not None: self.poller.start() self.kernel.start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: pass