class InternalIPKernel(object):
"""Class for managing an IPython kernel inside of QGIS.
IPython normally runs kernels in seperate processes, but this setup is
limiting in the case of QGIS because external kernels cannot access the
data and variables that QGIS is working with. This class manages an
in-process kernel and is capable of launching external consoles that can
attach to the kernel.
IPython Consoles are run as external processes because they rely on
Version 2 of the PyQt api and QGIS is using Version 1 which is
incompatible.
"""
def __init__(self):
self.ipkernel = IPKernelApp()
self.ipkernel.initialize(['python',
# Ensure the Kernel pre-loads the same modules as are available
# from the QGIS python console.
"--c='from qgis.core import *;import qgis.utils;'"])
# Ensure we use an event loop that is QGIS-friendly.
self.ipkernel.kernel.eventloop = loop_qgis
self.ipkernel.start()
# To create and track active qt consoles
self._qtconsole_cmd = ['ipython', 'qtconsole', '--existing'] + \
[os.path.basename(self.ipkernel.connection_file)]
self.consoles = []
def new_qt_console(self, evt=None):
self.consoles.append(subprocess.Popen(self._qtconsole_cmd))
def cleanup_consoles(self, evt=None):
for c in self.consoles:
c.kill()
def pylab_kernel(gui):
"""Launch and return an IPython kernel with pylab support for the desired gui
"""
kernel = IPKernelApp()
kernel.initialize(['python', '--pylab=%s' % gui,
#'--log-level=10'
])
return kernel
def pylab_kernel(gui):
"""Launch and return an IPython kernel with pylab support for the desired gui
"""
kernel = IPKernelApp()
# FIXME: we're hardcoding the heartbeat port b/c of a bug in IPython 0.11
# that will set it to 0 if not specified. Once this is fixed, the --hb
# parameter can be omitted and all ports will be automatic
kernel.initialize(['python', '--pylab=%s' % gui, '--hb=19999',
#'--log-level=10'
])
return kernel
def get_connection_file(app=None):
"""Return the path to the connection file of an app
Parameters
----------
app : KernelApp instance [optional]
If unspecified, the currently running app will be used
"""
if app is None:
from IPython.zmq.ipkernel import IPKernelApp
if not IPKernelApp.initialized():
raise RuntimeError("app not specified, and not in a running Kernel")
app = IPKernelApp.instance()
return filefind(app.connection_file, ['.', app.profile_dir.security_dir])
def init_ipython():
"""
Encapsulate Kernel creation logic: Only the kernel client, manager and shell are exposed
This is in order to ensure interoperability between major IPython changes
"""
if ipython_1:
manager = QtInProcessKernelManager()
manager.start_kernel()
manager.kernel.gui = 'qt4'
shell = manager.kernel.shell
shell.run_cell('%pylab inline')
client = manager.client()
client.start_channels()
else:
def event_loop(kernel):
kernel.timer = QTimer()
kernel.timer.timeout.connect(kernel.do_one_iteration)
kernel.timer.start(1000 * kernel._poll_interval)
kernel_app = IPKernelApp.instance()
kernel_app.initialize(['python', '--pylab=qt']) # at this point, print() won’t work anymore
kernel_app.kernel.eventloop = event_loop
connection_file = find_connection_file(kernel_app.connection_file)
manager = QtKernelManager(connection_file=connection_file)
manager.load_connection_file()
manager.start_channels()
kernel_app.start()
client = None
shell = kernel_app.shell
return client, manager, shell
def pylab_kernel(self,gui):
'''Launch an IPython kernel with pylab support for the gui.'''
log_debug,pdb,trace = False,True,False
tag = 'leoIPython.py:pylab_kernel'
kernel = IPKernelApp.instance()
# IPKernalApp is a singleton class.
# Return the singleton instance, creating it if necessary.
if kernel:
# pylab is needed for Qt event loop integration.
args = ['python','--pylab=%s' % (gui)]
if log_debug: args.append('--debug')
#'--log-level=10'
# '--pdb', # User-level debugging
try:
if pdb: g.pdb()
kernel.initialize(args)
# kernel objects: (Leo --ipython arg)
# kernel.session: zmq.session.Session
# kernel.shell: ZMQInteractiveShell
# kernel.shell.comm_manager: comm.manager.CommManager.
# kernel.shell.event = events.EventManager
# kernel.shell.run_cell (method)
# kernel.shell.hooks
except Exception:
sys.stdout = sys.__stdout__
print('%s: kernel.initialize failed!' % tag)
raise
else:
print('%s IPKernelApp.instance failed' % (tag))
return kernel
def bind_kernel(**kwargs):
"""Bind an Engine's Kernel to be used as a full IPython kernel.
This allows a running Engine to be used simultaneously as a full IPython kernel
with the QtConsole or other frontends.
This function returns immediately.
"""
from IPython.zmq.ipkernel import IPKernelApp
from IPython.parallel.apps.ipengineapp import IPEngineApp
# first check for IPKernelApp, in which case this should be a no-op
# because there is already a bound kernel
if IPKernelApp.initialized() and isinstance(IPKernelApp._instance, IPKernelApp):
return
if IPEngineApp.initialized():
try:
app = IPEngineApp.instance()
except MultipleInstanceError:
pass
else:
return app.bind_kernel(**kwargs)
raise RuntimeError("bind_kernel be called from an IPEngineApp instance")
def pylab_kernel(gui):
"""Launch and return an IPython kernel with pylab support for the desired gui
"""
kernel = IPKernelApp.instance()
# Note: pylab command seems to be needed for event loop to behave nicely
kernel.initialize([get_executable(), '--pylab=%s' % gui,
"--c='%run -m mantidplotrc'"])
return kernel
def pylab_kernel(gui):
"""Launch and return an IPython kernel with pylab support for the desired gui
"""
kernel = IPKernelApp.instance()
# pylab is really needed, for Qt event loop integration
kernel.initialize(['python', '--pylab=%s' % gui,
#'--log-level=10'
])
return kernel
def __init__(self):
self._app = IPKernelApp.instance()
self._app.initialize()
fh = logging.FileHandler(self.__class__.__name__ + ".log")
fh.setLevel(logging.DEBUG)
log = self._app.kernel.log
log.setLevel(logging.DEBUG)
log.addHandler(fh)
log.debug("kernel init")
def fork_kernel(self, config, pipe, resource_limits, logfile):
os.setpgrp()
logging.basicConfig(filename=self.filename,format=str(uuid.uuid4()).split('-')[0]+': %(asctime)s %(message)s',level=logging.DEBUG)
ka = IPKernelApp.instance(config=config, ip=config["ip"])
ka.initialize([])
if self.update_function is not None:
self.update_function(ka)
for r, limit in resource_limits.iteritems():
resource.setrlimit(getattr(resource, r), (limit, limit))
pipe.send({"ip": ka.ip, "key": ka.session.key, "shell_port": ka.shell_port,
"stdin_port": ka.stdin_port, "hb_port": ka.hb_port, "iopub_port": ka.iopub_port})
pipe.close()
ka.start()
def __init__(self):
self.ipkernel = IPKernelApp()
self.ipkernel.initialize(['python',
# Ensure the Kernel pre-loads the same modules as are available
# from the QGIS python console.
"--c='from qgis.core import *;import qgis.utils;'"])
# Ensure we use an event loop that is QGIS-friendly.
self.ipkernel.kernel.eventloop = loop_qgis
self.ipkernel.start()
# To create and track active qt consoles
self._qtconsole_cmd = ['ipython', 'qtconsole', '--existing'] + \
[os.path.basename(self.ipkernel.connection_file)]
self.consoles = []
def fork_kernel(self, sage_dict, config, q):
ka = IPKernelApp.instance(config=config)
ka.initialize([])
ka.kernel.shell.user_ns.update(sage_dict)
ka.kernel.shell.user_ns.update(interact.classes)
if "sys" in ka.kernel.shell.user_ns:
ka.kernel.shell.user_ns["sys"]._interacts = interact.interacts
else:
sys._interacts = interact.interacts
ka.kernel.shell.user_ns["sys"] = sys
ka.kernel.shell.user_ns["interact"] = interact.interact_func(ka.session, ka.iopub_socket)
q.send({"ip": ka.ip, "key": ka.session.key, "shell_port": ka.shell_port,
"stdin_port": ka.stdin_port, "hb_port": ka.hb_port, "iopub_port": ka.iopub_port})
q.close()
ka.start()
def pylab_kernel(self,gui):
"""Launch and return an IPython kernel with pylab support for the desired gui
"""
trace = True
tag = 'leoIPython.py'
kernel = IPKernelApp.instance()
if kernel:
# pylab is really needed, for Qt event loop integration.
try:
kernel.initialize(['python','--pylab=%s' % (gui)])
#'--log-level=10'
if trace: print('%s: kernel: %s' % (tag,kernel))
except Exception:
print('%s: kernel.initialize failed!' % tag)
raise
else:
print('%s IPKernelApp.instance failed' % (tag))
return kernel
def pylab_kernel(self,gui):
"""Launch and return an IPython kernel with pylab support for the desired gui
"""
trace = True
tag = 'leoIPython.py'
kernel = IPKernelApp.instance()
if kernel:
# pylab is really needed, for Qt event loop integration.
try:
kernel.initialize(['python','--pylab=%s' % (gui)])
#'--log-level=10'
if trace: print('%s: kernel: %s' % (tag,kernel))
except Exception:
print('%s: kernel.initialize failed!' % tag)
raise
else:
print('%s IPKernelApp.instance failed' % (tag))
return kernel
def __init__(self, gui, shell):
self.shell = shell
self.logger = None
if shell is None:
# Start IPython kernel with GUI event loop support
self.ipkernel = IPKernelApp.instance()
self.ipkernel.initialize(['python', '--gui=%s' % gui,
#'--log-level=10' # for debugging
])
# This application will also act on the shell user namespace
self.namespace = self.ipkernel.shell.user_ns
else:
self.ipkernel = shell.config.IPKernelApp
self.namespace = shell.user_ns
# To create and track active qt consoles
self.consoles = []
def __init__(self, gui):
# Start IPython kernel with GUI event loop support
self.ipkernel = IPKernelApp.instance()
self.ipkernel.initialize(['python', '--gui=%s' % gui,
#'--log-level=10' # for debugging
])
# To create and track active qt consoles
self.consoles = []
# This application will also act on the shell user namespace
self.namespace = self.ipkernel.shell.user_ns
# Keys present at startup so we don't print the entire pylab/numpy
# namespace when the user clicks the 'namespace' button
self._init_keys = set(self.namespace.keys())
# Example: a variable that will be seen by the user in the shell, and
# that the GUI modifies (the 'Counter++' button increments it):
self.namespace['app_counter'] = 0
def __init__(self, gui, shell):
self.shell = shell
self.logger = None
if shell is None:
# Start IPython kernel with GUI event loop support
self.ipkernel = IPKernelApp.instance()
self.ipkernel.initialize([
'python',
'--gui=%s' % gui,
#'--log-level=10' # for debugging
])
# This application will also act on the shell user namespace
self.namespace = self.ipkernel.shell.user_ns
else:
self.ipkernel = shell.config.IPKernelApp
self.namespace = shell.user_ns
# To create and track active qt consoles
self.consoles = []
def bind_kernel(self, **kwargs):
"""Promote engine to listening kernel, accessible to frontends."""
if self.kernel_app is not None:
return
self.log.info(
"Opening ports for direct connections as an IPython kernel")
kernel = self.kernel
kwargs.setdefault('config', self.config)
kwargs.setdefault('log', self.log)
kwargs.setdefault('profile_dir', self.profile_dir)
kwargs.setdefault('session', self.engine.session)
app = self.kernel_app = IPKernelApp(**kwargs)
# allow IPKernelApp.instance():
IPKernelApp._instance = app
app.init_connection_file()
# relevant contents of init_sockets:
app.shell_port = app._bind_socket(kernel.shell_streams[0],
app.shell_port)
app.log.debug("shell ROUTER Channel on port: %i", app.shell_port)
app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port)
app.log.debug("iopub PUB Channel on port: %i", app.iopub_port)
kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER)
app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port)
app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port)
# start the heartbeat, and log connection info:
app.init_heartbeat()
app.log_connection_info()
app.write_connection_file()
def fork_kernel(self, config, pipe, resource_limits, logfile):
os.setpgrp()
logging.basicConfig(filename=self.filename,
format=str(uuid.uuid4()).split('-')[0] +
': %(asctime)s %(message)s',
level=logging.DEBUG)
ka = IPKernelApp.instance(config=config, ip=config["ip"])
ka.initialize([])
if self.update_function is not None:
self.update_function(ka)
for r, limit in resource_limits.iteritems():
resource.setrlimit(getattr(resource, r), (limit, limit))
pipe.send({
"ip": ka.ip,
"key": ka.session.key,
"shell_port": ka.shell_port,
"stdin_port": ka.stdin_port,
"hb_port": ka.hb_port,
"iopub_port": ka.iopub_port
})
pipe.close()
ka.start()
def default_kernel_app():
""" Return a configured IPKernelApp """
def event_loop(kernel):
""" Non-blocking qt event loop."""
kernel.timer = QtCore.QTimer()
kernel.timer.timeout.connect(kernel.do_one_iteration)
kernel.timer.start(1000 * kernel._poll_interval)
app = IPKernelApp.instance()
try:
app.initialize(['python', '--pylab=qt'])
except ZMQError:
pass # already set up
app.kernel.eventloop = event_loop
try:
app.start()
except RuntimeError: # already started
pass
return app
def complete_registration(self, msg, connect, maybe_tunnel):
# print msg
self._abort_dc.stop()
ctx = self.context
loop = self.loop
identity = self.bident
idents, msg = self.session.feed_identities(msg)
msg = Message(self.session.unserialize(msg))
if msg.content.status == 'ok':
self.id = int(msg.content.id)
# launch heartbeat
hb_addrs = msg.content.heartbeat
# possibly forward hb ports with tunnels
hb_addrs = [maybe_tunnel(addr) for addr in hb_addrs]
heart = Heart(*map(str, hb_addrs), heart_id=identity)
heart.start()
# create Shell Streams (MUX, Task, etc.):
queue_addr = msg.content.mux
shell_addrs = [str(queue_addr)]
task_addr = msg.content.task
if task_addr:
shell_addrs.append(str(task_addr))
# Uncomment this to go back to two-socket model
# shell_streams = []
# for addr in shell_addrs:
# stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
# stream.setsockopt(zmq.IDENTITY, identity)
# stream.connect(disambiguate_url(addr, self.location))
# shell_streams.append(stream)
# Now use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
connect(stream, addr)
# end single stream-socket
# control stream:
control_addr = str(msg.content.control)
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
connect(control_stream, control_addr)
# create iopub stream:
iopub_addr = msg.content.iopub
iopub_socket = ctx.socket(zmq.PUB)
iopub_socket.setsockopt(zmq.IDENTITY, identity)
connect(iopub_socket, iopub_addr)
# disable history:
self.config.HistoryManager.hist_file = ':memory:'
# Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session,
iopub_socket, u'stdout')
sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id)
sys.stderr = self.out_stream_factory(self.session,
iopub_socket, u'stderr')
sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id)
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(
self.session, iopub_socket)
sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id)
self.kernel = Kernel(config=self.config,
int_id=self.id,
ident=self.ident,
session=self.session,
control_stream=control_stream,
shell_streams=shell_streams,
iopub_socket=iopub_socket,
loop=loop,
user_ns=self.user_ns,
log=self.log)
self.kernel.shell.display_pub.topic = cast_bytes(
'engine.%i.displaypub' % self.id)
# FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged
app = IPKernelApp(config=self.config,
shell=self.kernel.shell,
kernel=self.kernel,
log=self.log)
app.init_profile_dir()
app.init_code()
self.kernel.start()
else:
self.log.fatal("Registration Failed: %s" % msg)
raise Exception("Registration Failed: %s" % msg)
self.log.info("Completed registration with id %i" % self.id)
def kernel_target_f(config):
"""launch a kernel, for use with multiprocessing.Process(target=this)"""
app = IPKernelApp.instance(config=config)
app.initialize([])
app.start()
def kernel_target_f(config):
"""launch a kernel, for use with multiprocessing.Process(target=this)"""
app = IPKernelApp.instance(config=config)
app.initialize([])
app.start()
""" launching IPython from GDB! http://blog.scottt.tw/2012/01/exploring-gdb-python-api-with-ipython_31.html usage: 0) start gdb, eg: "gdb /usr/sbin/kopano-server core" 1) from gdb: run "source gdbi.py" -> this will provide a 'kernel identifier', eg "kernel-1234.json" 2) start ipython using eg: "ipython console --existing kernel-1234.json" 3) start specific script, eg: "import kopano_cache" """ app = IPKernelApp.instance() app.initialize([]) app.start()
# Interactive python from IPython.zmq.ipkernel import IPKernelApp app = IPKernelApp.instance() app.initialize([]) app.start()
def default_kernel_app():
app = IPKernelApp.instance()
app.initialize(['python', '--pylab=qt'])
app.kernel.eventloop = event_loop
return app
def pylab_kernel(gui):
"""Launch and return an IPython kernel with pylab support for the desired gui
"""
kernel = IPKernelApp.instance()
kernel.initialize(['python', '--pylab=%s' % gui, '--log-level=10'])
return kernel
try:
sys.path.remove(osp.dirname(__file__))
except ValueError:
pass
locals().pop('__file__')
__doc__ = ''
__name__ = '__main__'
if os.environ.get('IPYTHON_KERNEL', False):
# IPython >=v0.11 Kernel
# Fire up the kernel instance.
from IPython.zmq.ipkernel import IPKernelApp
ipk_temp = IPKernelApp.instance()
ipk_temp.initialize(sys.argv[1:])
__ipythonshell__ = ipk_temp.shell
# Issue 977: Since kernel.initialize() has completed execution,
# we can now allow the monitor to communicate the availablility of
# the kernel to accept front end connections.
__ipythonkernel__ = ipk_temp
del ipk_temp
# Start the (infinite) kernel event loop.
__ipythonkernel__.start()
elif os.environ.get('IPYTHON', False):
sys.path.insert(0, '')
def complete_registration(self, msg, connect, maybe_tunnel):
# print msg
self._abort_dc.stop()
ctx = self.context
loop = self.loop
identity = self.bident
idents, msg = self.session.feed_identities(msg)
msg = self.session.unserialize(msg)
content = msg['content']
info = self.connection_info
def url(key):
"""get zmq url for given channel"""
return str(info["interface"] + ":%i" % info[key])
if content['status'] == 'ok':
self.id = int(content['id'])
# launch heartbeat
# possibly forward hb ports with tunnels
hb_ping = maybe_tunnel(url('hb_ping'))
hb_pong = maybe_tunnel(url('hb_pong'))
heart = Heart(hb_ping, hb_pong, heart_id=identity)
heart.start()
# create Shell Connections (MUX, Task, etc.):
shell_addrs = url('mux'), url('task')
# Use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
connect(stream, addr)
# control stream:
control_addr = url('control')
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
connect(control_stream, control_addr)
# create iopub stream:
iopub_addr = url('iopub')
iopub_socket = ctx.socket(zmq.PUB)
iopub_socket.setsockopt(zmq.IDENTITY, identity)
connect(iopub_socket, iopub_addr)
# disable history:
self.config.HistoryManager.hist_file = ':memory:'
# Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session,
iopub_socket, u'stdout')
sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id)
sys.stderr = self.out_stream_factory(self.session,
iopub_socket, u'stderr')
sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id)
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(
self.session, iopub_socket)
sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id)
self.kernel = Kernel(config=self.config,
int_id=self.id,
ident=self.ident,
session=self.session,
control_stream=control_stream,
shell_streams=shell_streams,
iopub_socket=iopub_socket,
loop=loop,
user_ns=self.user_ns,
log=self.log)
self.kernel.shell.display_pub.topic = cast_bytes(
'engine.%i.displaypub' % self.id)
# FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged
app = IPKernelApp(config=self.config,
shell=self.kernel.shell,
kernel=self.kernel,
log=self.log)
app.init_profile_dir()
app.init_code()
self.kernel.start()
else:
self.log.fatal("Registration Failed: %s" % msg)
raise Exception("Registration Failed: %s" % msg)
self.log.info("Completed registration with id %i" % self.id)
try:
sys.path.remove(osp.dirname(__file__))
except ValueError:
pass
locals().pop('__file__')
__doc__ = ''
__name__ = '__main__'
if os.environ.get('IPYTHON_KERNEL', False):
# IPython >=v0.11 Kernel
from IPython.zmq.ipkernel import IPKernelApp
__ipythonkernel__ = IPKernelApp().instance()
__ipythonkernel__.initialize(sys.argv[1:])
__ipythonshell__ = __ipythonkernel__.shell
__ipythonkernel__.start()
elif os.environ.get('IPYTHON', False):
sys.path.insert(0, '')
if os.name == 'nt':
# Windows platforms: monkey-patching *pyreadline* module
# to make IPython work in a remote process
from pyreadline import unicode_helper
unicode_helper.pyreadline_codepage = "ascii"
# For pyreadline >= v1.7:
from pyreadline import rlmain
class Readline(rlmain.Readline):
def run(self): self.app=IPKernelApp.instance() self.app.initialize() self.app.start()
def default_kernel_app():
app = IPKernelApp.instance()
app.initialize(['python', '--pylab=qt'])
app.kernel.eventloop = event_loop
return app
except ValueError:
pass
locals().pop('__file__')
__doc__ = ''
__name__ = '__main__'
if os.environ.get('IPYTHON_KERNEL', False):
# IPython >=v0.11 Kernel
# Fire up the kernel instance.
from IPython.zmq.ipkernel import IPKernelApp
ipk_temp = IPKernelApp.instance()
ipk_temp.initialize(sys.argv[1:])
__ipythonshell__ = ipk_temp.shell
# Issue 977: Since kernel.initialize() has completed execution,
# we can now allow the monitor to communicate the availablility of
# the kernel to accept front end connections.
__ipythonkernel__ = ipk_temp
del ipk_temp
# Start the (infinite) kernel event loop.
__ipythonkernel__.start()
elif os.environ.get('IPYTHON', False):
sys.path.insert(0, '')
def complete_registration(self, msg, connect, maybe_tunnel):
# print msg
self._abort_dc.stop()
ctx = self.context
loop = self.loop
identity = self.bident
idents,msg = self.session.feed_identities(msg)
msg = self.session.unserialize(msg)
content = msg['content']
info = self.connection_info
def url(key):
"""get zmq url for given channel"""
return str(info["interface"] + ":%i" % info[key])
if content['status'] == 'ok':
self.id = int(content['id'])
# launch heartbeat
# possibly forward hb ports with tunnels
hb_ping = maybe_tunnel(url('hb_ping'))
hb_pong = maybe_tunnel(url('hb_pong'))
hb_monitor = None
if self.max_heartbeat_misses > 0:
# Add a monitor socket which will record the last time a ping was seen
mon = self.context.socket(zmq.SUB)
mport = mon.bind_to_random_port('tcp://127.0.0.1')
mon.setsockopt(zmq.SUBSCRIBE, b"")
self._hb_listener = zmqstream.ZMQStream(mon, self.loop)
self._hb_listener.on_recv(self._report_ping)
hb_monitor = "tcp://127.0.0.1:%i"%mport
heart = Heart(hb_ping, hb_pong, hb_monitor , heart_id=identity)
heart.start()
# create Shell Connections (MUX, Task, etc.):
shell_addrs = url('mux'), url('task')
# Use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
connect(stream, addr)
# control stream:
control_addr = url('control')
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
connect(control_stream, control_addr)
# create iopub stream:
iopub_addr = url('iopub')
iopub_socket = ctx.socket(zmq.PUB)
iopub_socket.setsockopt(zmq.IDENTITY, identity)
connect(iopub_socket, iopub_addr)
# disable history:
self.config.HistoryManager.hist_file = ':memory:'
# Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout')
sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id)
sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr')
sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id)
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(self.session, iopub_socket)
sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id)
self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session,
control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket,
loop=loop, user_ns=self.user_ns, log=self.log)
self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id)
# periodically check the heartbeat pings of the controller
# Should be started here and not in "start()" so that the right period can be taken
# from the hubs HeartBeatMonitor.period
if self.max_heartbeat_misses > 0:
# Use a slightly bigger check period than the hub signal period to not warn unnecessary
self.hb_check_period = int(content['hb_period'])+10
self.log.info("Starting to monitor the heartbeat signal from the hub every %i ms." , self.hb_check_period)
self._hb_reporter = ioloop.PeriodicCallback(self._hb_monitor, self.hb_check_period, self.loop)
self._hb_reporter.start()
else:
self.log.info("Monitoring of the heartbeat signal from the hub is not enabled.")
# FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged
app = IPKernelApp(config=self.config, shell=self.kernel.shell, kernel=self.kernel, log=self.log)
app.init_profile_dir()
app.init_code()
self.kernel.start()
else:
self.log.fatal("Registration Failed: %s"%msg)
raise Exception("Registration Failed: %s"%msg)
self.log.info("Completed registration with id %i"%self.id)
def _start_kernel():
"""starts the ipython kernel and returns the ipython app"""
from IPython.zmq.ipkernel import IPKernelApp
from zmq.eventloop import ioloop
global _kernel_running, _ipython_app
if _kernel_running:
return _ipython_app
# get the app if it exists, or set it up if it doesn't
if IPKernelApp.initialized():
app = IPKernelApp.instance()
else:
app = IPKernelApp.instance()
app.initialize()
# Undo unnecessary sys module mangling from init_sys_modules.
# This would not be necessary if we could prevent it
# in the first place by using a different InteractiveShell
# subclass, as in the regular embed case.
main = app.kernel.shell._orig_sys_modules_main_mod
if main is not None:
sys.modules[app.kernel.shell._orig_sys_modules_main_name] = main
app.kernel.user_module = sys.modules[__name__]
app.kernel.user_ns = {}
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
old_set_completer_frame = InteractiveShell.set_completer_frame
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(app.shell, InteractiveShell)
app.shell.set_completer_frame = bound_scf
app.shell.set_completer_frame()
# start the kernel
app.kernel.start()
# set up a timer to periodically poll the zmq ioloop
loop = ioloop.IOLoop.instance()
def poll_ioloop(timer_id, time):
global _kernel_running
# if the kernel has been closed then run the event loop until it gets to the
# stop event added by IPKernelApp.shutdown_request
if app.kernel.shell.exit_now:
_log.debug("IPython kernel stopping (%s)" % app.connection_file)
timer.kill_timer(timer_id)
ioloop.IOLoop.instance().start()
_kernel_running = False
return
# otherwise call the event loop but stop immediately if there are no pending events
loop.add_timeout(0, lambda: loop.add_callback(loop.stop))
ioloop.IOLoop.instance().start()
_log.debug("IPython kernel starting. Use '--existing %s' to connect." % app.connection_file)
timer.set_timer(100, poll_ioloop)
_kernel_running = True
_ipython_app = app
return _ipython_app
def complete_registration(self, msg, connect, maybe_tunnel):
# print msg
self._abort_dc.stop()
ctx = self.context
loop = self.loop
identity = self.bident
idents,msg = self.session.feed_identities(msg)
msg = self.session.unserialize(msg)
content = msg['content']
info = self.connection_info
def url(key):
"""get zmq url for given channel"""
return str(info["interface"] + ":%i" % info[key])
if content['status'] == 'ok':
self.id = int(content['id'])
# launch heartbeat
# possibly forward hb ports with tunnels
hb_ping = maybe_tunnel(url('hb_ping'))
hb_pong = maybe_tunnel(url('hb_pong'))
heart = Heart(hb_ping, hb_pong, heart_id=identity)
heart.start()
# create Shell Connections (MUX, Task, etc.):
shell_addrs = url('mux'), url('task')
# Use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
connect(stream, addr)
# control stream:
control_addr = url('control')
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
connect(control_stream, control_addr)
# create iopub stream:
iopub_addr = url('iopub')
iopub_socket = ctx.socket(zmq.PUB)
iopub_socket.setsockopt(zmq.IDENTITY, identity)
connect(iopub_socket, iopub_addr)
# disable history:
self.config.HistoryManager.hist_file = ':memory:'
# Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout')
sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id)
sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr')
sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id)
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(self.session, iopub_socket)
sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id)
self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session,
control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket,
loop=loop, user_ns=self.user_ns, log=self.log)
self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id)
# FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged
app = IPKernelApp(config=self.config, shell=self.kernel.shell, kernel=self.kernel, log=self.log)
app.init_profile_dir()
app.init_code()
self.kernel.start()
else:
self.log.fatal("Registration Failed: %s"%msg)
raise Exception("Registration Failed: %s"%msg)
self.log.info("Completed registration with id %i"%self.id)
def _start_kernel():
"""starts the ipython kernel and returns the ipython app"""
from IPython.zmq.ipkernel import IPKernelApp
from zmq.eventloop import ioloop
global _kernel_running, _ipython_app
if _kernel_running:
return _ipython_app
# get the app if it exists, or set it up if it doesn't
if IPKernelApp.initialized():
app = IPKernelApp.instance()
else:
app = IPKernelApp.instance()
app.initialize()
# Undo unnecessary sys module mangling from init_sys_modules.
# This would not be necessary if we could prevent it
# in the first place by using a different InteractiveShell
# subclass, as in the regular embed case.
main = app.kernel.shell._orig_sys_modules_main_mod
if main is not None:
sys.modules[app.kernel.shell._orig_sys_modules_main_name] = main
app.kernel.user_module = sys.modules[__name__]
app.kernel.user_ns = {}
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
old_set_completer_frame = InteractiveShell.set_completer_frame
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(app.shell, InteractiveShell)
app.shell.set_completer_frame = bound_scf
app.shell.set_completer_frame()
# start the kernel
app.kernel.start()
# set up a timer to periodically poll the zmq ioloop
loop = ioloop.IOLoop.instance()
def poll_ioloop(timer_id, time):
global _kernel_running
# if the kernel has been closed then run the event loop until it gets to the
# stop event added by IPKernelApp.shutdown_request
if app.kernel.shell.exit_now:
_log.debug("IPython kernel stopping (%s)" % app.connection_file)
timer.kill_timer(timer_id)
ioloop.IOLoop.instance().start()
_kernel_running = False
return
# otherwise call the event loop but stop immediately if there are no pending events
loop.add_timeout(0, lambda: loop.add_callback(loop.stop))
ioloop.IOLoop.instance().start()
_log.debug("IPython kernel starting. Use '--existing %s' to connect." % app.connection_file)
timer.set_timer(100, poll_ioloop)
_kernel_running = True
_ipython_app = app
return _ipython_app
def complete_registration(self, msg, connect, maybe_tunnel):
# print msg
self._abort_dc.stop()
ctx = self.context
loop = self.loop
identity = self.bident
idents,msg = self.session.feed_identities(msg)
msg = Message(self.session.unserialize(msg))
if msg.content.status == 'ok':
self.id = int(msg.content.id)
# launch heartbeat
hb_addrs = msg.content.heartbeat
# possibly forward hb ports with tunnels
hb_addrs = [ maybe_tunnel(addr) for addr in hb_addrs ]
heart = Heart(*list(map(str, hb_addrs)), heart_id=identity)
heart.start()
# create Shell Streams (MUX, Task, etc.):
queue_addr = msg.content.mux
shell_addrs = [ str(queue_addr) ]
task_addr = msg.content.task
if task_addr:
shell_addrs.append(str(task_addr))
# Uncomment this to go back to two-socket model
# shell_streams = []
# for addr in shell_addrs:
# stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
# stream.setsockopt(zmq.IDENTITY, identity)
# stream.connect(disambiguate_url(addr, self.location))
# shell_streams.append(stream)
# Now use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
connect(stream, addr)
# end single stream-socket
# control stream:
control_addr = str(msg.content.control)
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
connect(control_stream, control_addr)
# create iopub stream:
iopub_addr = msg.content.iopub
iopub_socket = ctx.socket(zmq.PUB)
iopub_socket.setsockopt(zmq.IDENTITY, identity)
connect(iopub_socket, iopub_addr)
# disable history:
self.config.HistoryManager.hist_file = ':memory:'
# Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session, iopub_socket, 'stdout')
sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id)
sys.stderr = self.out_stream_factory(self.session, iopub_socket, 'stderr')
sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id)
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(self.session, iopub_socket)
sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id)
self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session,
control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket,
loop=loop, user_ns=self.user_ns, log=self.log)
self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id)
# FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged
app = IPKernelApp(config=self.config, shell=self.kernel.shell, kernel=self.kernel, log=self.log)
app.init_profile_dir()
app.init_code()
self.kernel.start()
else:
self.log.fatal("Registration Failed: %s"%msg)
raise Exception("Registration Failed: %s"%msg)
self.log.info("Completed registration with id %i"%self.id)
