def test_io_api(): """Test that wrapped stdout has the same API as a normal TextIO object""" session = Session() ctx = zmq.Context() pub = ctx.socket(zmq.PUB) thread = IOPubThread(pub) thread.start() stream = OutStream(session, thread, 'stdout') # cleanup unused zmq objects before we start testing thread.stop() thread.close() ctx.term() assert stream.errors is None assert not stream.isatty() with nt.assert_raises(io.UnsupportedOperation): stream.detach() with nt.assert_raises(io.UnsupportedOperation): next(stream) with nt.assert_raises(io.UnsupportedOperation): stream.read() with nt.assert_raises(io.UnsupportedOperation): stream.readline() with nt.assert_raises(io.UnsupportedOperation): stream.seek() with nt.assert_raises(io.UnsupportedOperation): stream.tell()
def test_io_api(): """Test that wrapped stdout has the same API as a normal TextIO object""" session = Session() ctx = zmq.Context() pub = ctx.socket(zmq.PUB) thread = IOPubThread(pub) thread.start() stream = OutStream(session, thread, 'stdout') # cleanup unused zmq objects before we start testing thread.stop() thread.close() ctx.term() assert stream.errors is None assert not stream.isatty() with pytest.raises(io.UnsupportedOperation): stream.detach() with pytest.raises(io.UnsupportedOperation): next(stream) with pytest.raises(io.UnsupportedOperation): stream.read() with pytest.raises(io.UnsupportedOperation): stream.readline() with pytest.raises(io.UnsupportedOperation): stream.seek(0) with pytest.raises(io.UnsupportedOperation): stream.tell() with pytest.raises(TypeError): stream.write(b'')
def test_io_isatty(): session = Session() ctx = zmq.Context() pub = ctx.socket(zmq.PUB) thread = IOPubThread(pub) thread.start() stream = OutStream(session, thread, 'stdout', isatty=True) assert stream.isatty()
def complete_registration(self, msg, connect, maybe_tunnel): # print msg self.loop.remove_timeout(self._abort_timeout) ctx = self.context loop = self.loop identity = self.bident idents, msg = self.session.feed_identities(msg) msg = self.session.deserialize(msg) content = msg['content'] info = self.connection_info def url(key): """get zmq url for given channel""" return str(info["interface"] + ":%i" % info[key]) if content['status'] == 'ok': if self.id is not None and content['id'] != self.id: self.log.warning("Did not get the requested id: %i != %i", content['id'], self.id) self.id = content['id'] # launch heartbeat # possibly forward hb ports with tunnels hb_ping = maybe_tunnel(url('hb_ping')) hb_pong = maybe_tunnel(url('hb_pong')) hb_monitor = None if self.max_heartbeat_misses > 0: # Add a monitor socket which will record the last time a ping was seen mon = self.context.socket(zmq.SUB) mport = mon.bind_to_random_port('tcp://%s' % localhost()) mon.setsockopt(zmq.SUBSCRIBE, b"") self._hb_listener = zmqstream.ZMQStream(mon, self.loop) self._hb_listener.on_recv(self._report_ping) hb_monitor = "tcp://%s:%i" % (localhost(), mport) heart = Heart(hb_ping, hb_pong, hb_monitor, heart_id=identity) heart.start() # create Shell Connections (MUX, Task, etc.): shell_addrs = url('mux'), url('task') # Use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) stream.setsockopt(zmq.IDENTITY, identity) shell_streams = [stream] for addr in shell_addrs: connect(stream, addr) # control stream: control_addr = url('control') control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) control_stream.setsockopt(zmq.IDENTITY, identity) connect(control_stream, control_addr) # create iopub stream: iopub_addr = url('iopub') iopub_socket = ctx.socket(zmq.PUB) iopub_socket.setsockopt(zmq.IDENTITY, identity) connect(iopub_socket, iopub_addr) try: from ipykernel.iostream import IOPubThread except ImportError: pass else: iopub_socket = IOPubThread(iopub_socket) iopub_socket.start() # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: sys.displayhook = self.display_hook_factory( self.session, iopub_socket) sys.displayhook.topic = cast_bytes('engine.%i.execute_result' % self.id) self.kernel = Kernel(parent=self, engine_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.shell.display_pub.topic = cast_bytes( 'engine.%i.displaypub' % self.id) # periodically check the heartbeat pings of the controller # Should be started here and not in "start()" so that the right period can be taken # from the hubs HeartBeatMonitor.period if self.max_heartbeat_misses > 0: # Use a slightly bigger check period than the hub signal period to not warn unnecessary self.hb_check_period = int(content['hb_period']) + 10 self.log.info( "Starting to monitor the heartbeat signal from the hub every %i ms.", self.hb_check_period) self._hb_reporter = ioloop.PeriodicCallback( self._hb_monitor, self.hb_check_period) self._hb_reporter.start() else: self.log.info( "Monitoring of the heartbeat signal from the hub is not enabled." ) # FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged app = IPKernelApp(parent=self, shell=self.kernel.shell, kernel=self.kernel, log=self.log) if self.use_mpi and self.init_mpi: app.exec_lines.insert(0, self.init_mpi) app.init_profile_dir() app.init_code() self.kernel.start() else: self.log.fatal("Registration Failed: %s" % msg) raise Exception("Registration Failed: %s" % msg) self.log.info("Completed registration with id %i" % self.id)
def _complete_registration(self, msg, connect, maybe_tunnel): ctx = self.context loop = self.loop identity = self.bident idents, msg = self.session.feed_identities(msg) msg = self.session.deserialize(msg) content = msg['content'] info = self.connection_info def url(key): """get zmq url for given channel""" return str(info["interface"] + ":%i" % info[key]) def urls(key): return [f'{info["interface"]}:{port}' for port in info[key]] if content['status'] == 'ok': requested_id = self.id self.id = content['id'] if requested_id is not None and self.id != requested_id: self.log.warning( f"Did not get the requested id: {self.id} != {requested_id}" ) self.log.name = self.log.name.rsplit(".", 1)[0] + f".{self.id}" elif self.id is None: self.log.name += f".{self.id}" # create Shell Connections (MUX, Task, etc.): # select which broadcast endpoint to connect to # use rank % len(broadcast_leaves) broadcast_urls = urls('broadcast') broadcast_leaves = len(broadcast_urls) broadcast_index = self.id % len(broadcast_urls) broadcast_url = broadcast_urls[broadcast_index] shell_addrs = [url('mux'), url('task'), broadcast_url] self.log.info(f'Shell_addrs: {shell_addrs}') # Use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) stream.setsockopt(zmq.IDENTITY, identity) # TODO: enable PROBE_ROUTER when schedulers can handle the empty message # stream.setsockopt(zmq.PROBE_ROUTER, 1) self.log.debug("Setting shell identity %r", identity) shell_streams = [stream] for addr in shell_addrs: self.log.info("Connecting shell to %s", addr) connect(stream, addr) # control stream: control_url = url('control') curve_serverkey = self.curve_serverkey if self.enable_nanny: nanny_url, self.nanny_pipe = self.start_nanny( control_url=control_url, ) control_url = nanny_url # nanny uses our curve_publickey, not the controller's publickey curve_serverkey = self.curve_publickey control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) control_stream.setsockopt(zmq.IDENTITY, identity) connect(control_stream, control_url, curve_serverkey=curve_serverkey) # create iopub stream: iopub_addr = url('iopub') iopub_socket = ctx.socket(zmq.PUB) iopub_socket.SNDHWM = 0 iopub_socket.setsockopt(zmq.IDENTITY, identity) connect(iopub_socket, iopub_addr) try: from ipykernel.iostream import IOPubThread except ImportError: pass else: iopub_socket = IOPubThread(iopub_socket) iopub_socket.start() # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, 'stdout') sys.stdout.topic = f"engine.{self.id}.stdout".encode("ascii") sys.stderr = self.out_stream_factory(self.session, iopub_socket, 'stderr') sys.stderr.topic = f"engine.{self.id}.stderr".encode("ascii") # copied from ipykernel 6, which captures sys.__stderr__ at the FD-level if getattr(sys.stderr, "_original_stdstream_copy", None) is not None: for handler in self.log.handlers: if isinstance(handler, StreamHandler) and ( handler.stream.buffer.fileno() == 2): self.log.debug( "Seeing logger to stderr, rerouting to raw filedescriptor." ) handler.stream = TextIOWrapper( FileIO(sys.stderr._original_stdstream_copy, "w")) if self.display_hook_factory: sys.displayhook = self.display_hook_factory( self.session, iopub_socket) sys.displayhook.topic = f"engine.{self.id}.execute_result".encode( "ascii") # patch Session to always send engine uuid metadata original_send = self.session.send def send_with_metadata( stream, msg_or_type, content=None, parent=None, ident=None, buffers=None, track=False, header=None, metadata=None, **kwargs, ): """Ensure all messages set engine uuid metadata""" metadata = metadata or {} metadata.setdefault("engine", self.ident) return original_send( stream, msg_or_type, content=content, parent=parent, ident=ident, buffers=buffers, track=track, header=header, metadata=metadata, **kwargs, ) self.session.send = send_with_metadata self.kernel = Kernel.instance( parent=self, engine_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log, ) self.kernel.shell.display_pub.topic = f"engine.{self.id}.displaypub".encode( "ascii") # FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged self.init_signal() app = IPKernelApp(parent=self, shell=self.kernel.shell, kernel=self.kernel, log=self.log) if self.use_mpi and self.init_mpi: app.exec_lines.insert(0, self.init_mpi) app.init_profile_dir() app.init_code() self.kernel.start() else: self.log.fatal("Registration Failed: %s" % msg) raise Exception("Registration Failed: %s" % msg) self.start_heartbeat( maybe_tunnel(url('hb_ping')), maybe_tunnel(url('hb_pong')), content['hb_period'], identity, ) self.log.info("Completed registration with id %i" % self.id) self.loop.remove_timeout(self._abort_timeout)
def complete_registration(self, msg, connect, maybe_tunnel): # print msg self.loop.remove_timeout(self._abort_timeout) ctx = self.context loop = self.loop identity = self.bident idents,msg = self.session.feed_identities(msg) msg = self.session.deserialize(msg) content = msg['content'] info = self.connection_info def url(key): """get zmq url for given channel""" return str(info["interface"] + ":%i" % info[key]) if content['status'] == 'ok': if self.id is not None and content['id'] != self.id: self.log.warning("Did not get the requested id: %i != %i", content['id'], self.id) self.id = content['id'] # launch heartbeat # possibly forward hb ports with tunnels hb_ping = maybe_tunnel(url('hb_ping')) hb_pong = maybe_tunnel(url('hb_pong')) hb_monitor = None if self.max_heartbeat_misses > 0: # Add a monitor socket which will record the last time a ping was seen mon = self.context.socket(zmq.SUB) mport = mon.bind_to_random_port('tcp://%s' % localhost()) mon.setsockopt(zmq.SUBSCRIBE, b"") self._hb_listener = zmqstream.ZMQStream(mon, self.loop) self._hb_listener.on_recv(self._report_ping) hb_monitor = "tcp://%s:%i" % (localhost(), mport) heart = Heart(hb_ping, hb_pong, hb_monitor , heart_id=identity) heart.start() # create Shell Connections (MUX, Task, etc.): shell_addrs = url('mux'), url('task') # Use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) stream.setsockopt(zmq.IDENTITY, identity) shell_streams = [stream] for addr in shell_addrs: connect(stream, addr) # control stream: control_addr = url('control') control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) control_stream.setsockopt(zmq.IDENTITY, identity) connect(control_stream, control_addr) # create iopub stream: iopub_addr = url('iopub') iopub_socket = ctx.socket(zmq.PUB) iopub_socket.setsockopt(zmq.IDENTITY, identity) connect(iopub_socket, iopub_addr) try: from ipykernel.iostream import IOPubThread except ImportError: pass else: iopub_socket = IOPubThread(iopub_socket) iopub_socket.start() # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: sys.displayhook = self.display_hook_factory(self.session, iopub_socket) sys.displayhook.topic = cast_bytes('engine.%i.execute_result' % self.id) self.kernel = Kernel(parent=self, engine_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id) # periodically check the heartbeat pings of the controller # Should be started here and not in "start()" so that the right period can be taken # from the hubs HeartBeatMonitor.period if self.max_heartbeat_misses > 0: # Use a slightly bigger check period than the hub signal period to not warn unnecessary self.hb_check_period = int(content['hb_period'])+10 self.log.info("Starting to monitor the heartbeat signal from the hub every %i ms." , self.hb_check_period) self._hb_reporter = ioloop.PeriodicCallback(self._hb_monitor, self.hb_check_period, self.loop) self._hb_reporter.start() else: self.log.info("Monitoring of the heartbeat signal from the hub is not enabled.") # FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged app = IPKernelApp(parent=self, shell=self.kernel.shell, kernel=self.kernel, log=self.log) if self.use_mpi and self.init_mpi: app.exec_lines.insert(0, self.init_mpi) app.init_profile_dir() app.init_code() self.kernel.start() else: self.log.fatal("Registration Failed: %s"%msg) raise Exception("Registration Failed: %s"%msg) self.log.info("Completed registration with id %i"%self.id)
class IPKernelServer(BaseIPythonApplication, InteractiveShellApp, ConnectionFileMixin): name = 'ipython-kernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session] # the kernel class, as an importstring kernel_class = Type('ipykernel.ipkernel.IPythonKernel', klass='ipykernel.kernelbase.Kernel', help="""The Kernel subclass to be used. This should allow easy re-use of the IPKernelApp entry point to configure and launch kernels other than IPython's own. """).tag(config=True) kernel = Any() poller = Any( ) # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat, allow_none=True) ports = Dict() subcommands = { 'install': ('ipykernel.kernelspec.InstallIPythonKernelSpecApp', 'Install the IPython kernel'), } # connection info: connection_dir = Unicode() @default('connection_dir') def _default_connection_dir(self): return jupyter_runtime_dir() @property def abs_connection_file(self): if os.path.basename(self.connection_file) == self.connection_file: return os.path.join(self.connection_dir, self.connection_file) else: return self.connection_file # streams, etc. no_stdout = Bool( False, help="redirect stdout to the null device").tag(config=True) no_stderr = Bool( False, help="redirect stderr to the null device").tag(config=True) quiet = Bool( True, help="Only send stdout/stderr to output stream").tag(config=True) outstream_class = DottedObjectName( 'ipykernel.iostream.OutStream', help="The importstring for the OutStream factory").tag(config=True) #displayhook_class = DottedObjectName('ipykernel.displayhook.ZMQDisplayHook', # help="The importstring for the DisplayHook factory").tag(config=True) # polling parent_handle = Integer( int(os.environ.get('JPY_PARENT_PID') or 0), help="""kill this process if its parent dies. On Windows, the argument specifies the HANDLE of the parent process, otherwise it is simply boolean. """).tag(config=True) interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0), help="""ONLY USED ON WINDOWS Interrupt this process when the parent is signaled. """).tag(config=True) def init_crash_handler(self): sys.excepthook = self.excepthook def excepthook(self, etype, evalue, tb): # write uncaught traceback to 'real' stderr, not zmq-forwarder traceback.print_exception(etype, evalue, tb, file=sys.__stderr__) def init_poller(self): if sys.platform == 'win32': if self.interrupt or self.parent_handle: self.poller = ParentPollerWindows(self.interrupt, self.parent_handle) elif self.parent_handle and self.parent_handle != 1: # PID 1 (init) is special and will never go away, # only be reassigned. # Parent polling doesn't work if ppid == 1 to start with. self.poller = ParentPollerUnix() def _bind_socket(self, s, port): iface = '%s://%s' % (self.transport, self.ip) if self.transport == 'tcp': if port <= 0: port = s.bind_to_random_port(iface) else: s.bind("tcp://%s:%i" % (self.ip, port)) elif self.transport == 'ipc': if port <= 0: port = 1 path = "%s-%i" % (self.ip, port) while os.path.exists(path): port = port + 1 path = "%s-%i" % (self.ip, port) else: path = "%s-%i" % (self.ip, port) s.bind("ipc://%s" % path) return port def write_connection_file(self): """write connection info to JSON file""" cf = self.abs_connection_file self.log.debug("Writing connection file: %s", cf) write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port, control_port=self.control_port) def cleanup_connection_file(self): cf = self.abs_connection_file self.log.debug("Cleaning up connection file: %s", cf) try: os.remove(cf) except (IOError, OSError): pass self.cleanup_ipc_files() def init_connection_file(self): if not self.connection_file: self.connection_file = "kernel-%s.json" % os.getpid() try: self.connection_file = filefind(self.connection_file, ['.', self.connection_dir]) except IOError: self.log.debug("Connection file not found: %s", self.connection_file) # This means I own it, and I'll create it in this directory: ensure_dir_exists(os.path.dirname(self.abs_connection_file), 0o700) # Also, I will clean it up: atexit.register(self.cleanup_connection_file) return try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_sockets(self): # Create a context, a session, and the kernel sockets. self.log.info("Starting the kernel at pid: %i", os.getpid()) context = zmq.Context.instance() # Uncomment this to try closing the context. # atexit.register(context.term) self.shell_socket = context.socket(zmq.ROUTER) self.shell_socket.linger = 1000 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_socket.linger = 1000 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) self.control_socket = context.socket(zmq.ROUTER) self.control_socket.linger = 1000 self.control_port = self._bind_socket(self.control_socket, self.control_port) self.log.debug("control ROUTER Channel on port: %i" % self.control_port) if hasattr(zmq, 'ROUTER_HANDOVER'): # set router-handover to workaround zeromq reconnect problems # in certain rare circumstances # see ipython/ipykernel#270 and zeromq/libzmq#2892 self.shell_socket.router_handover = \ self.control_socket.router_handover = \ self.stdin_socket.router_handover = 1 self.init_iopub(context) def init_iopub(self, context): self.iopub_socket = context.socket(zmq.PUB) self.iopub_socket.linger = 1000 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) self.configure_tornado_logger() self.iopub_thread = IOPubThread(self.iopub_socket, pipe=True) self.iopub_thread.start() # backward-compat: wrap iopub socket API in background thread self.iopub_socket = self.iopub_thread.background_socket def init_heartbeat(self): """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) self.heartbeat.start() def log_connection_info(self): """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.connection_dir: # use shortname tail = basename else: tail = self.connection_file lines = [ "To connect another client to this kernel, use:", " --existing %s" % tail, ] # log connection info # info-level, so often not shown. # frontends should use the %connect_info magic # to see the connection info for line in lines: self.log.info(line) # also raw print to the terminal if no parent_handle (`ipython kernel`) # unless log-level is CRITICAL (--quiet) if not self.parent_handle and self.log_level < logging.CRITICAL: io.rprint(_ctrl_c_message) for line in lines: io.rprint(line) self.ports = dict(shell=self.shell_port, iopub=self.iopub_port, stdin=self.stdin_port, hb=self.hb_port, control=self.control_port) def init_blackhole(self): """redirects stdout/stderr to devnull if necessary""" if self.no_stdout or self.no_stderr: blackhole = open(os.devnull, 'w') if self.no_stdout: sys.stdout = sys.__stdout__ = blackhole if self.no_stderr: sys.stderr = sys.__stderr__ = blackhole def init_io(self): """Redirect input streams and set a display hook.""" if self.outstream_class: outstream_factory = import_item(str(self.outstream_class)) sys.stdout.flush() e_stdout = None if self.quiet else sys.__stdout__ e_stderr = None if self.quiet else sys.__stderr__ sys.stdout = outstream_factory(self.session, self.iopub_thread, u'stdout', echo=e_stdout) sys.stderr.flush() sys.stderr = outstream_factory(self.session, self.iopub_thread, u'stderr', echo=e_stderr) if self.displayhook_class: displayhook_factory = import_item(str(self.displayhook_class)) self.displayhook = displayhook_factory(self.session, self.iopub_socket) sys.displayhook = self.displayhook self.patch_io() def patch_io(self): """Patch important libraries that can't handle sys.stdout forwarding""" try: import faulthandler except ImportError: pass else: # Warning: this is a monkeypatch of `faulthandler.enable`, watch for possible # updates to the upstream API and update accordingly (up-to-date as of Python 3.5): # https://docs.python.org/3/library/faulthandler.html#faulthandler.enable # change default file to __stderr__ from forwarded stderr faulthandler_enable = faulthandler.enable def enable(file=sys.__stderr__, all_threads=True, **kwargs): return faulthandler_enable(file=file, all_threads=all_threads, **kwargs) faulthandler.enable = enable if hasattr(faulthandler, 'register'): faulthandler_register = faulthandler.register def register(signum, file=sys.__stderr__, all_threads=True, chain=False, **kwargs): return faulthandler_register(signum, file=file, all_threads=all_threads, chain=chain, **kwargs) faulthandler.register = register def init_signal(self): signal.signal(signal.SIGINT, signal.SIG_IGN) def init_kernel(self): """Create the Kernel object itself""" shell_stream = ZMQStream(self.shell_socket) control_stream = ZMQStream(self.control_socket) kernel_factory = self.kernel_class.instance kernel = kernel_factory( parent=self, session=self.session, shell_streams=[shell_stream, control_stream], iopub_thread=self.iopub_thread, iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, user_ns=self.user_ns, ) kernel.record_ports( {name + '_port': port for name, port in self.ports.items()}) self.kernel = kernel # Allow the displayhook to get the execution count #self.displayhook.get_execution_count = lambda: kernel.execution_count def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" # Register inline backend as default # this is higher priority than matplotlibrc, # but lower priority than anything else (mpl.use() for instance). # This only affects matplotlib >= 1.5 if not os.environ.get('MPLBACKEND'): os.environ[ 'MPLBACKEND'] = 'module://ipykernel.pylab.backend_inline' # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` # to ensure that any exception is printed straight to stderr. # Normally _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. shell = self.shell _showtraceback = shell._showtraceback try: # replace error-sending traceback with stderr def print_tb(etype, evalue, stb): print("GUI event loop or pylab initialization failed", file=sys.stderr) print(shell.InteractiveTB.stb2text(stb), file=sys.stderr) shell._showtraceback = print_tb InteractiveShellApp.init_gui_pylab(self) finally: shell._showtraceback = _showtraceback def init_shell(self): self.shell = getattr(self.kernel, 'shell', None) if self.shell: self.shell.configurables.append(self) def init_extensions(self): super(IPKernelApp, self).init_extensions() # BEGIN HARDCODED WIDGETS HACK # Ensure ipywidgets extension is loaded if available extension_man = self.shell.extension_manager if 'ipywidgets' not in extension_man.loaded: try: extension_man.load_extension('ipywidgets') except ImportError as e: self.log.debug( 'ipywidgets package not installed. Widgets will not be available.' ) # END HARDCODED WIDGETS HACK def configure_tornado_logger(self): """ Configure the tornado logging.Logger. Must set up the tornado logger or else tornado will call basicConfig for the root logger which makes the root logger go to the real sys.stderr instead of the capture streams. This function mimics the setup of logging.basicConfig. """ logger = logging.getLogger('tornado') handler = logging.StreamHandler() formatter = logging.Formatter(logging.BASIC_FORMAT) handler.setFormatter(formatter) logger.addHandler(handler) @catch_config_error def initialize(self, argv=None): super(IPKernelServer, self).initialize(argv) if self.subapp is not None: return # register zmq IOLoop with tornado zmq_ioloop.install() self.init_blackhole() self.init_connection_file() self.init_poller() self.init_sockets() self.init_heartbeat() # writing/displaying connection info must be *after* init_sockets/heartbeat self.write_connection_file() # Log connection info after writing connection file, so that the connection # file is definitely available at the time someone reads the log. self.log_connection_info() #self.init_io() self.init_signal() self.init_kernel() # shell init steps self.init_path() #self.init_shell() #if self.shell: # self.init_gui_pylab() # self.init_extensions() # self.init_code() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request #sys.stdout.flush() #sys.stderr.flush() def start(self): if self.subapp is not None: return self.subapp.start() if self.poller is not None: self.poller.start() self.kernel.start() self.io_loop = ioloop.IOLoop.current() try: self.io_loop.start() except KeyboardInterrupt: pass
class YAPKernelApp(BaseIPythonApplication, InteractiveShellApp, ConnectionFileMixin): name='YAP-kernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [YAPKernel, ZMQInteractiveShell, ProfileDir, Session] # the kernel class, as an importstring kernel_class = Type('yap_kernel.yap_kernel.YAPKernel', klass='ipykernel.kernelbase.Kernel', help="""The Kernel subclass to be used. This should allow easy re-use of the YAPKernelApp entry point to configure and launch kernels other than YAP's own. """).tag(config=True) kernel = Any() poller = Any() # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat, allow_none=True) ports = Dict() subcommands = { 'install': ( 'yap_kernel.kernelspec.InstallYAPKernelSpecApp', 'Install the YAP kernel' ), } # connection info: connection_dir = Unicode() @default('connection_dir') def _default_connection_dir(self): return jupyter_runtime_dir() @property def abs_connection_file(self): if os.path.basename(self.connection_file) == self.connection_file: return os.path.join(self.connection_dir, self.connection_file) else: return self.connection_file # streams, etc. no_stdout = Bool(False, help="redirect stdout to the null device").tag(config=True) no_stderr = Bool(False, help="redirect stderr to the null device").tag(config=True) outstream_class = DottedObjectName('ipykernel.iostream.OutStream', help="The importstring for the OutStream factory").tag(config=True) displayhook_class = DottedObjectName('ipykernel.displayhook.ZMQDisplayHook', help="The importstring for the DisplayHook factory").tag(config=True) # polling parent_handle = Integer(int(os.environ.get('JPY_PARENT_PID') or 0), help="""kill this process if its parent dies. On Windows, the argument specifies the HANDLE of the parent process, otherwise it is simply boolean. """).tag(config=True) interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0), help="""ONLY USED ON WINDOWS Interrupt this process when the parent is signaled. """).tag(config=True) def init_crash_handler(self): sys.excepthook = self.excepthook def excepthook(self, etype, evalue, tb): # write uncaught traceback to 'real' stderr, not zmq-forwarder traceback.print_exception(etype, evalue, tb, file=sys.__stderr__) def init_poller(self): if sys.platform == 'win32': if self.interrupt or self.parent_handle: self.poller = ParentPollerWindows(self.interrupt, self.parent_handle) elif self.parent_handle: self.poller = ParentPollerUnix() def _bind_socket(self, s, port): iface = '%s://%s' % (self.transport, self.ip) if self.transport == 'tcp': if port <= 0: port = s.bind_to_random_port(iface) else: s.bind("tcp://%s:%i" % (self.ip, port)) elif self.transport == 'ipc': if port <= 0: port = 1 path = "%s-%i" % (self.ip, port) while os.path.exists(path): port = port + 1 path = "%s-%i" % (self.ip, port) else: path = "%s-%i" % (self.ip, port) s.bind("ipc://%s" % path) return port def write_connection_file(self): """write connection info to JSON file""" cf = self.abs_connection_file self.log.debug("Writing connection file: %s", cf) write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port, control_port=self.control_port) def cleanup_connection_file(self): cf = self.abs_connection_file self.log.debug("Cleaning up connection file: %s", cf) try: os.remove(cf) except (IOError, OSError): pass self.cleanup_ipc_files() def init_connection_file(self): if not self.connection_file: self.connection_file = "kernel-%s.json"%os.getpid() try: self.connection_file = filefind(self.connection_file, ['.', self.connection_dir]) except IOError: self.log.debug("Connection file not found: %s", self.connection_file) # This means I own it, and I'll create it in this directory: ensure_dir_exists(os.path.dirname(self.abs_connection_file), 0o700) # Also, I will clean it up: atexit.register(self.cleanup_connection_file) return try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_sockets(self): # Create a context, a session, and the kernel sockets. self.log.info("Starting the kernel at pid: %i", os.getpid()) context = zmq.Context.instance() # Uncomment this to try closing the context. # atexit.register(context.term) self.shell_socket = context.socket(zmq.ROUTER) self.shell_socket.linger = 1000 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_socket.linger = 1000 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) self.control_socket = context.socket(zmq.ROUTER) self.control_socket.linger = 1000 self.control_port = self._bind_socket(self.control_socket, self.control_port) self.log.debug("control ROUTER Channel on port: %i" % self.control_port) self.init_iopub(context) def init_iopub(self, context): self.iopub_socket = context.socket(zmq.PUB) self.iopub_socket.linger = 1000 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) self.configure_tornado_logger() self.iopub_thread = IOPubThread(self.iopub_socket, pipe=True) self.iopub_thread.start() # backward-compat: wrap iopub socket API in background thread self.iopub_socket = self.iopub_thread.background_socket def init_heartbeat(self): """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) self.heartbeat.start() def log_connection_info(self): """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.connection_dir: # use shortname tail = basename else: tail = self.connection_file lines = [ "To connect another client to this kernel, use:", " --existing %s" % tail, ] # log connection info # info-level, so often not shown. # frontends should use the %connect_info magic # to see the connection info for line in lines: self.log.info(line) # also raw print to the terminal if no parent_handle (`ipython kernel`) # unless log-level is CRITICAL (--quiet) if not self.parent_handle and self.log_level < logging.CRITICAL: io.rprint(_ctrl_c_message) for line in lines: io.rprint(line) self.ports = dict(shell=self.shell_port, iopub=self.iopub_port, stdin=self.stdin_port, hb=self.hb_port, control=self.control_port) def init_blackhole(self): """redirects stdout/stderr to devnull if necessary""" if self.no_stdout or self.no_stderr: blackhole = open(os.devnull, 'w') if self.no_stdout: sys.stdout = sys.__stdout__ = blackhole if self.no_stderr: sys.stderr = sys.__stderr__ = blackhole def init_io(self): """Redirect input streams and set a display hook.""" if self.outstream_class: outstream_factory = import_item(str(self.outstream_class)) sys.stdout = outstream_factory(self.session, self.iopub_thread, u'stdout') sys.stderr = outstream_factory(self.session, self.iopub_thread, u'stderr') if self.displayhook_class: displayhook_factory = import_item(str(self.displayhook_class)) self.displayhook = displayhook_factory(self.session, self.iopub_socket) sys.displayhook = self.displayhook self.patch_io() def patch_io(self): """Patch important libraries that can't handle sys.stdout forwarding""" try: import faulthandler except ImportError: pass else: # Warning: this is a monkeypatch of `faulthandler.enable`, watch for possible # updates to the upstream API and update accordingly (up-to-date as of Python 3.5): # https://docs.python.org/3/library/faulthandler.html#faulthandler.enable # change default file to __stderr__ from forwarded stderr faulthandler_enable = faulthandler.enable def enable(file=sys.__stderr__, all_threads=True, **kwargs): return faulthandler_enable(file=file, all_threads=all_threads, **kwargs) faulthandler.enable = enable if hasattr(faulthandler, 'register'): faulthandler_register = faulthandler.register def register(signum, file=sys.__stderr__, all_threads=True, chain=False, **kwargs): return faulthandler_register(signum, file=file, all_threads=all_threads, chain=chain, **kwargs) faulthandler.register = register def init_signal(self): signal.signal(signal.SIGINT, signal.SIG_IGN) def init_kernel(self): """Create the Kernel object itself""" shell_stream = ZMQStream(self.shell_socket) control_stream = ZMQStream(self.control_socket) kernel_factory = self.kernel_class.instance kernel = kernel_factory(parent=self, session=self.session, shell_streams=[shell_stream, control_stream], iopub_thread=self.iopub_thread, iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, user_ns=self.user_ns, ) kernel.record_ports({ name + '_port': port for name, port in self.ports.items() }) self.kernel = kernel # Allow the displayhook to get the execution count self.displayhook.get_execution_count = lambda: kernel.execution_count def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" # Register inline backend as default # this is higher priority than matplotlibrc, # but lower priority than anything else (mpl.use() for instance). # This only affects matplotlib >= 1.5 if not os.environ.get('MPLBACKEND'): os.environ['MPLBACKEND'] = 'module://ipykernel.pylab.backend_inline' # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` # to ensure that any exception is printed straight to stderr. # Normally _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. shell = self.shell _showtraceback = shell._showtraceback try: # replace error-sending traceback with stderr def print_tb(etype, evalue, stb): print ("GUI event loop or pylab initialization failed", file=sys.stderr) print (shell.InteractiveTB.stb2text(stb), file=sys.stderr) shell._showtraceback = print_tb InteractiveShellApp.init_gui_pylab(self) finally: shell._showtraceback = _showtraceback def init_shell(self): self.shell = getattr(self.kernel, 'shell', None) if self.shell: self.shell.configurables.append(self) def init_extensions(self): super(YAPKernelApp, self).init_extensions() # BEGIN HARDCODED WIDGETS HACK # Ensure ipywidgets extension is loaded if available extension_man = self.shell.extension_manager if 'ipywidgets' not in extension_man.loaded: try: extension_man.load_extension('ipywidgets') except ImportError as e: self.log.debug('ipywidgets package not installed. Widgets will not be available.') # END HARDCODED WIDGETS HACK def configure_tornado_logger(self): """ Configure the tornado logging.Logger. Must set up the tornado logger or else tornado will call basicConfig for the root logger which makes the root logger go to the real sys.stderr instead of the capture streams. This function mimics the setup of logging.basicConfig. """ logger = logging.getLogger('tornado') handler = logging.StreamHandler() formatter = logging.Formatter(logging.BASIC_FORMAT) handler.setFormatter(formatter) logger.addHandler(handler) @catch_config_error def initialize(self, argv=None): super(YAPKernelApp, self).initialize(argv) if self.subapp is not None: return # register zmq IOLoop with tornado zmq_ioloop.install() self.init_blackhole() self.init_connection_file() self.init_poller() self.init_sockets() self.init_heartbeat() # writing/displaying connection info must be *after* init_sockets/heartbeat self.write_connection_file() # Log connection info after writing connection file, so that the connection # file is definitely available at the time someone reads the log. self.log_connection_info() self.init_io() self.init_signal() self.init_kernel() # shell init steps self.init_path() self.init_shell() if self.shell: self.init_gui_pylab() self.init_extensions() self.init_code() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request sys.stdout.flush() sys.stderr.flush() def start(self): if self.subapp is not None: return self.subapp.start() if self.poller is not None: self.poller.start() self.kernel.start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: pass