def fork_debugger(namespace=None):
logger.warning('forking with ipython kernel for debugging ...')
try:
from IPython import embed_kernel, get_ipython, Config, config, terminal
from IPython.kernel.zmq import kernelapp
curr = get_ipython()
if curr:
logger.warning("there's a current ipython running (%r)", curr)
for cls in terminal.interactiveshell.TerminalInteractiveShell.mro():
if hasattr(cls, 'clear_instance'):
cls.clear_instance()
for cls in kernelapp.IPKernelApp.mro():
if hasattr(cls, 'clear_instance'):
cls.clear_instance()
conf = Config()
conf.InteractiveShellApp.code_to_run = 'raise'
if namespace:
conf.IPKernelApp.connection_file = '{}/kernel.json'.format(
namespace.namespace())
logger.warning('starting ipython for debugging (%s)', namespace)
embed_kernel(config=conf)
logger.warning('embeding finished with debugging (%s)', namespace)
except Exception as e:
logger.error('failed to embed ipython: %s', e, exc_info=True)
def run_kernel():
from IPython import release
if release.version_info[0] < 2:
print(self.style.ERROR("--kernel requires at least IPython version 2.0"))
return
from IPython import embed_kernel
imported_objects = import_objects(options, self.style)
embed_kernel(local_ns=imported_objects)
def run_ipython_kernel(globals, locals):
from IPython import embed_kernel
class DummyMod:
pass
user_module = DummyMod()
user_module.__dict__ = globals
embed_kernel(module=user_module, local_ns=locals)
def run_kernel():
from IPython import release
if release.version_info[0] < 2:
print(
self.style.ERROR(
"--kernel requires at least IPython version 2.0"))
return
from IPython import embed_kernel
imported_objects = import_objects(options, self.style)
embed_kernel(local_ns=imported_objects)
def __init__(self, **kwargs):
self.provider = IMAPProvider()
self.logged_in = False
self.status = urwid.Text('Status: ')
self.stack = [LoginView(self)]
self.frame = urwid.Frame(self.stack[-1], self.status)
self.loop = urwid.MainLoop(self.frame,
unhandled_input=self.unhandled_input)
if 'debug' in kwargs:
if kwargs['debug']:
embed_kernel()
def setup_ipython_embed(shell_api=None):
from gevent_zeromq import monkey_patch
monkey_patch()
# patch in device:
# gevent-zeromq does not support devices, which block in the C layer.
# we need to support the "heartbeat" which is a simple bounceback, so we
# simulate it using the following method.
import zmq
orig_device = zmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == zmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device.device(dev_type, insock, outsock, *args)
zmq.device = device_patch
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
from IPython.zmq.ipkernel import IPKernelApp
old_start = IPKernelApp.start
old_set_completer_frame = InteractiveShell.set_completer_frame
def new_start(appself):
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(
appself.shell, InteractiveShell)
appself.shell.set_completer_frame = bound_scf
appself.shell.set_completer_frame()
old_start(appself)
IPKernelApp.start = new_start
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile #, shutil
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config.Application.ipython_dir = temp_dir
with patch(
"IPython.core.interactiveshell.InteractiveShell.init_virtualenv"
):
embed_kernel(local_ns=shell_api,
config=ipy_config) # blocks until INT
def __init__(self, **kwargs):
self.provider = IMAPProvider()
self.logged_in = False
self.status = urwid.Text('Status: ')
self.stack = [LoginView(self)]
self.frame = urwid.Frame(self.stack[-1],
self.status)
self.loop = urwid.MainLoop(self.frame,
unhandled_input=self.unhandled_input)
if 'debug' in kwargs:
if kwargs['debug']:
embed_kernel()
def setup_ipython_embed(shell_api=None):
_assert_ipython_dir()
from gevent_zeromq import monkey_patch
monkey_patch()
# patch in device:
# gevent-zeromq does not support devices, which block in the C layer.
# we need to support the "heartbeat" which is a simple bounceback, so we
# simulate it using the following method.
import zmq
orig_device = zmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == zmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device.device(dev_type, insock, outsock, *args)
zmq.device = device_patch
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
from IPython.zmq.ipkernel import IPKernelApp
old_start = IPKernelApp.start
old_set_completer_frame = InteractiveShell.set_completer_frame
def new_start(appself):
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(appself.shell, InteractiveShell)
appself.shell.set_completer_frame = bound_scf
appself.shell.set_completer_frame()
old_start(appself)
IPKernelApp.start = new_start
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile#, shutil
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config.Application.ipython_dir = temp_dir
with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
embed_kernel(local_ns=shell_api, config=ipy_config) # blocks until INT
def handle_noargs(self, **options):
imported_objects = get_import_objects()
config = None
prompt_prefix = get_prompt_prefix()
if prompt_prefix:
config = load_default_config()
config.PromptManager.in_template = (
'(%s) In [\#]: ' % prompt_prefix)
if options['kernel']:
from IPython import embed_kernel
embed_kernel(local_n=imported_objects, config=config)
else:
embed(user_ns=imported_objects, config=config)
def start_ipython(namespace, cluster_type="spark", **kwargs):
from IPython import embed_kernel
# create an initial list of variables to clear
# we do this without deleting to preserve the locals so that
# initialize_namespace isn't affected by this mutation
to_delete = [k for k in namespace if not k.startswith('__')]
# initialize the namespace with the proper variables
initialize_namespace(namespace, cluster_type=cluster_type)
# delete the extraneous variables
for k in to_delete:
del namespace[k]
# Start the kernel
embed_kernel(local_ns=namespace, **kwargs)
def start_kernel_and_console():
import os
import zmq
pid = os.fork()
if pid == 0:
# child
from IPython import embed_kernel
dict_ = globals()
import inspect
frame = inspect.currentframe()
local_ns = frame.f_back.f_locals
dict_.update(local_ns)
embed_kernel(local_ns=dict_)
else:
import time
time.sleep(2)
from jupyter_console.app import main
main(argv=["--existing"])
def setup_ipython_embed(shell_api=None):
if not _exists_ipython_dir():
log.warn("IPython profile dir not found. Attempting to avoid race condition")
import gevent
import random
gevent.sleep(random.random() * 3.0) # Introduce a random delay to make conflict less likely
from gevent_zeromq import monkey_patch
monkey_patch()
# patch in device:
# gevent-zeromq does not support devices, which block in the C layer.
# we need to support the "heartbeat" which is a simple bounceback, so we
# simulate it using the following method.
import zmq
orig_device = zmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == zmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device.device(dev_type, insock, outsock, *args)
zmq.device = device_patch
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
from IPython.zmq.ipkernel import IPKernelApp
old_start = IPKernelApp.start
old_set_completer_frame = InteractiveShell.set_completer_frame
def new_start(appself):
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(appself.shell, InteractiveShell)
appself.shell.set_completer_frame = bound_scf
appself.shell.set_completer_frame()
old_start(appself)
IPKernelApp.start = new_start
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile # , shutil
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config.Application.ipython_dir = temp_dir
with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
for tries in range(3):
try:
embed_kernel(local_ns=shell_api, config=ipy_config) # blocks until INT
break
except Exception as ex:
log.debug("Failed IPython initialize attempt (try #%s): %s", tries, str(ex))
import gevent
import random
gevent.sleep(random.random() * 0.5)
def open_ipython_kernel(self, message, namespace):
from IPython import embed_kernel
embed_kernel(local_ns=namespace)
def run_kernel():
imported_objects = import_objects(options, self.style)
embed_kernel(local_ns=imported_objects)
def activate_ipython_console(n):
from IPython import embed_kernel
embed_kernel(local_ns={'uwsgi': uwsgi})
def setup_ipython_embed(shell_api=None):
if not _exists_ipython_dir():
log.warn("IPython profile dir not found. Attempting to avoid race condition")
import gevent
import random
gevent.sleep(random.random() * 3.0) # Introduce a random delay to make conflict less likely
# Monkey patch zmq:
# There used to be gevent-zeromq, but it got included without monkey patch into zmq.
# Redo monkey patching, so that IPython's implicit use of zmq is green.
import zmq.green as gzmq
import zmq as ozmq
ozmq.Socket = gzmq.Socket
ozmq.Context = gzmq.Context
ozmq.Poller = gzmq.Poller
oioloop = __import__('zmq.eventloop.ioloop')
oioloop.Poller = gzmq.Poller
# Patch device:
# zmq.green.device still blocks in the C layer. We need to support the "heartbeat"
# which is a simple bounceback, so we simulate it using the following method.
orig_device = gzmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == ozmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device(dev_type, insock, outsock, *args)
ozmq.device = device_patch
gzmq.device = device_patch
from IPython.kernel.zmq import kernelapp
kernelapp._ctrl_c_message = "Ctrl-C is disabled. To end the process, use Ctrl-\\ or kill"
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config["config"].Application.ipython_dir = temp_dir
with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
for tries in range(3):
try:
embed_kernel(local_ns=shell_api, **ipy_config) # blocks until INT signal
break
except Exception as ex:
log.debug("Failed IPython initialize attempt (try #%s): %s", tries, str(ex))
import gevent
import random
gevent.sleep(random.random() * 0.5)
except:
try:
if os.path.exists(ipy_config["connection_file"]):
os.remove(ipy_config["connection_file"])
except Exception:
pass
raise
def activate_ipython_console(n):
from IPython import embed_kernel
embed_kernel(local_ns={'uwsgi': uwsgi})
def run_kernel():
imported_objects = import_objects(options, self.style)
embed_kernel(local_ns=imported_objects)
def open_ipython_kernel(self, message, namespace):
from IPython import embed_kernel
embed_kernel(local_ns=namespace)
def setup_ipython_embed(shell_api=None):
if not _exists_ipython_dir():
log.warn(
"IPython profile dir not found. Attempting to avoid race condition"
)
import gevent
import random
gevent.sleep(
random.random() *
3.0) # Introduce a random delay to make conflict less likely
from gevent_zeromq import monkey_patch
monkey_patch()
# patch in device:
# gevent-zeromq does not support devices, which block in the C layer.
# we need to support the "heartbeat" which is a simple bounceback, so we
# simulate it using the following method.
import zmq
orig_device = zmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == zmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device.device(dev_type, insock, outsock, *args)
zmq.device = device_patch
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
from IPython.zmq.ipkernel import IPKernelApp
old_start = IPKernelApp.start
old_set_completer_frame = InteractiveShell.set_completer_frame
def new_start(appself):
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(
appself.shell, InteractiveShell)
appself.shell.set_completer_frame = bound_scf
appself.shell.set_completer_frame()
old_start(appself)
IPKernelApp.start = new_start
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile #, shutil
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config.Application.ipython_dir = temp_dir
with patch(
"IPython.core.interactiveshell.InteractiveShell.init_virtualenv"
):
for tries in range(3):
try:
embed_kernel(local_ns=shell_api,
config=ipy_config) # blocks until INT signal
break
except Exception as ex:
log.debug(
"Failed IPython initialize attempt (try #%s): %s",
tries, str(ex))
import gevent
import random
gevent.sleep(random.random() * 0.5)
except:
try:
if os.path.exists(
ipy_config.KernelApp.connection_file):
os.remove(ipy_config.KernelApp.connection_file)
except Exception:
pass
raise
ip = "0.0.0.0"
# If the connection file doesn't exist, then create it.
if not os.path.isfile(connection_file):
key = str_to_bytes(str(uuid.uuid4()))
connection_file = determine_connection_file(connection_file)
write_connection_file(fname=connection_file, ip=ip, key=key)
if response_addr:
gateway_socket = return_connection_info(connection_file, ip,
response_addr,
disable_gateway_socket)
if gateway_socket: # socket in use, start gateway listener thread
gateway_listener_thread = Thread(target=gateway_listener,
args=(gateway_socket, ))
gateway_listener_thread.start()
# start to initialize the Spark session in the background
thread_to_initialize_spark_session.start()
# launch the IPython kernel instance
embed_kernel(connection_file=connection_file, ip=ip)
try:
os.remove(connection_file)
except:
pass
# stop the SparkContext after the kernel is stopped/killed
spark.stop()
# If the connection file doesn't exist, then create it.
if (connection_file and not os.path.isfile(connection_file)) or kernel_id is not None:
key = str_to_bytes(str(uuid.uuid4()))
connection_file = determine_connection_file(connection_file, kernel_id)
ports = _select_ports(5, lower_port, upper_port)
write_connection_file(fname=connection_file, ip=ip, key=key, shell_port=ports[0], iopub_port=ports[1],
stdin_port=ports[2], hb_port=ports[3], control_port=ports[4])
if response_addr:
gateway_socket = return_connection_info(connection_file, response_addr, lower_port, upper_port)
if gateway_socket: # socket in use, start gateway listener thread
gateway_listener_thread = Thread(target=gateway_listener, args=(gateway_socket,))
gateway_listener_thread.start()
# Initialize the kernel namespace for the given cluster type
if cluster_type == 'spark' and spark_init_mode == 'none':
cluster_type = 'none'
namespace = initialize_namespace(cluster_type=cluster_type)
# launch the IPython kernel instance
embed_kernel(local_ns=namespace,
connection_file=connection_file,
ip=ip)
try:
os.remove(connection_file)
except Exception:
pass
if not os.path.isfile(connection_file):
key = str_to_bytes(str(uuid.uuid4()))
connection_file = determine_connection_file(connection_file)
ports = _select_ports(5, lower_port, upper_port)
write_connection_file(fname=connection_file, ip=ip, key=key, shell_port=ports[0], iopub_port=ports[1],
stdin_port=ports[2], hb_port=ports[3], control_port=ports[4])
if response_addr:
gateway_socket = return_connection_info(connection_file, response_addr, disable_gateway_socket,
lower_port, upper_port)
if gateway_socket: # socket in use, start gateway listener thread
gateway_listener_thread = Thread(target=gateway_listener, args=(gateway_socket,))
gateway_listener_thread.start()
# start to initialize the Spark session in the background
if create_spark_context:
thread_to_initialize_spark_session.start()
# launch the IPython kernel instance
embed_kernel(connection_file=connection_file, ip=ip)
try:
os.remove(connection_file)
except:
pass
# stop the SparkContext after the kernel is stopped/killed
if create_spark_context:
spark.stop()
def setup_ipython_embed(shell_api=None):
if not _exists_ipython_dir():
log.warn(
"IPython profile dir not found. Attempting to avoid race condition"
)
import gevent
import random
gevent.sleep(
random.random() *
3.0) # Introduce a random delay to make conflict less likely
# Monkey patch zmq:
# There used to be gevent-zeromq, but it got included without monkey patch into zmq.
# Redo monkey patching, so that IPython's implicit use of zmq is green.
import zmq.green as gzmq
import zmq as ozmq
ozmq.Socket = gzmq.Socket
ozmq.Context = gzmq.Context
ozmq.Poller = gzmq.Poller
oioloop = __import__('zmq.eventloop.ioloop')
oioloop.Poller = gzmq.Poller
# Patch device:
# zmq.green.device still blocks in the C layer. We need to support the "heartbeat"
# which is a simple bounceback, so we simulate it using the following method.
orig_device = gzmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == ozmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device(dev_type, insock, outsock, *args)
ozmq.device = device_patch
gzmq.device = device_patch
from IPython.kernel.zmq import kernelapp
kernelapp._ctrl_c_message = "Ctrl-C is disabled. To end the process, use Ctrl-\\ or kill"
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config["config"].Application.ipython_dir = temp_dir
with patch(
"IPython.core.interactiveshell.InteractiveShell.init_virtualenv"
):
for tries in range(3):
try:
embed_kernel(local_ns=shell_api,
**ipy_config) # blocks until INT signal
break
except Exception as ex:
log.debug(
"Failed IPython initialize attempt (try #%s): %s",
tries, str(ex))
import gevent
import random
gevent.sleep(random.random() * 0.5)
except:
try:
if os.path.exists(ipy_config["connection_file"]):
os.remove(ipy_config["connection_file"])
except Exception:
pass
raise