def execute_helper(self,
code,
timeout=TIMEOUT,
silent=False,
store_history=True,
stop_on_error=True):
msg_id = self.kc.execute(code=code,
silent=silent,
store_history=store_history,
stop_on_error=stop_on_error)
reply = self.get_non_kernel_info_reply(timeout=timeout)
validate_message(reply, "execute_reply", msg_id)
busy_msg = run_sync(self.kc.iopub_channel.get_msg)(timeout=1)
validate_message(busy_msg, "status", msg_id)
self.assertEqual(busy_msg["content"]["execution_state"], "busy")
output_msgs = []
while True:
msg = run_sync(self.kc.iopub_channel.get_msg)(timeout=0.1)
validate_message(msg, msg["msg_type"], msg_id)
if msg["msg_type"] == "status":
self.assertEqual(msg["content"]["execution_state"], "idle")
break
elif msg["msg_type"] == "execute_input":
self.assertEqual(msg["content"]["code"], code)
continue
output_msgs.append(msg)
return reply, output_msgs
def execute_helper(self,
code,
timeout=TIMEOUT,
silent=False,
store_history=True,
stop_on_error=True):
msg_id = self.kc.execute(code=code,
silent=silent,
store_history=store_history,
stop_on_error=stop_on_error)
reply = self.get_non_kernel_info_reply(timeout=timeout)
validate_message(reply, 'execute_reply', msg_id)
busy_msg = run_sync(self.kc.iopub_channel.get_msg)(timeout=1)
validate_message(busy_msg, 'status', msg_id)
self.assertEqual(busy_msg['content']['execution_state'], 'busy')
output_msgs = []
while True:
msg = run_sync(self.kc.iopub_channel.get_msg)(timeout=0.1)
validate_message(msg, msg['msg_type'], msg_id)
if msg['msg_type'] == 'status':
self.assertEqual(msg['content']['execution_state'], 'idle')
break
elif msg['msg_type'] == 'execute_input':
self.assertEqual(msg['content']['code'], code)
continue
output_msgs.append(msg)
return reply, output_msgs
def handle_sigint(signum, frame):
# First proxy signal to subkernel, then run original handler
try:
run_sync(km.signal_kernel)(signum)
except RuntimeError as exc:
LOG.error(
f"Failed to interrupt subkernel {binding_name}: {exc}")
binding.update_progress("Idle")
orig_handler(signum, frame)
def test_run_sync_clean_up_task(loop):
async def coro_never_called():
pytest.fail("The call to this coroutine is not expected")
# Ensure that run_sync cancels the pending task
with mock.patch.object(loop, "run_until_complete") as patched_loop:
patched_loop.side_effect = KeyboardInterrupt
with mock.patch("asyncio.ensure_future") as patched_ensure_future:
mock_future = mock.Mock()
patched_ensure_future.return_value = mock_future
with pytest.raises(KeyboardInterrupt):
run_sync(coro_never_called)()
mock_future.cancel.assert_called_once()
# Suppress 'coroutine ... was never awaited' warning
patched_ensure_future.call_args[0][0].close()
def handle_execute_reply(self, msg_id, timeout=None):
kwargs = {"timeout": timeout}
msg = run_sync(self.client.shell_channel.get_msg)(**kwargs)
if msg["parent_header"].get("msg_id", None) == msg_id:
self.handle_iopub(msg_id)
content = msg["content"]
status = content['status']
if status == "aborted":
sys.stdout.write("Aborted\n")
return
elif status == 'ok':
# handle payloads
for item in content.get("payload", []):
source = item['source']
if source == 'page':
page.page(item['data']['text/plain'])
elif source == 'set_next_input':
self.next_input = item['text']
elif source == 'ask_exit':
self.keepkernel = item.get('keepkernel', False)
self.ask_exit()
elif status == 'error':
pass
self.execution_count = int(content["execution_count"] + 1)
def run_cell(self, cell, store_history=True):
"""Run a complete IPython cell.
Parameters
----------
cell : str
The code (including IPython code such as %magic functions) to run.
store_history : bool
If True, the raw and translated cell will be stored in IPython's
history. For user code calling back into IPython's machinery, this
should be set to False.
"""
if (not cell) or cell.isspace():
# pressing enter flushes any pending display
self.handle_iopub()
return
# flush stale replies, which could have been ignored, due to missed heartbeats
while run_sync(self.client.shell_channel.msg_ready)():
run_sync(self.client.shell_channel.get_msg)()
# execute takes 'hidden', which is the inverse of store_hist
msg_id = self.client.execute(cell, not store_history)
# first thing is wait for any side effects (output, stdin, etc.)
self._executing = True
self._execution_state = "busy"
while self._execution_state != 'idle' and self.client.is_alive():
try:
self.handle_input_request(msg_id, timeout=0.05)
except Empty:
# display intermediate print statements, etc.
self.handle_iopub(msg_id)
except ZMQError as e:
# Carry on if polling was interrupted by a signal
if e.errno != errno.EINTR:
raise
# after all of that is done, wait for the execute reply
while self.client.is_alive():
try:
self.handle_execute_reply(msg_id, timeout=0.05)
except Empty:
pass
else:
break
self._executing = False
def flush_channels(self):
for channel in (self.kc.shell_channel, self.kc.iopub_channel):
while True:
try:
msg = run_sync(channel.get_msg)(timeout=0.1)
except Empty:
break
else:
validate_message(msg)
def start(self):
"""Start the polling of the kernel."""
if self._pcallback is None:
if asyncio.iscoroutinefunction(self.poll):
cb = run_sync(self.poll)
else:
cb = self.poll
self._pcallback = ioloop.PeriodicCallback(
self.poll, 1000*self.time_to_dead,
)
self._pcallback.start()
def handle_input_request(self, msg_id, timeout=0.1):
""" Method to capture raw_input
"""
req = run_sync(self.client.stdin_channel.get_msg)(timeout=timeout)
# in case any iopub came while we were waiting:
self.handle_iopub(msg_id)
if msg_id == req["parent_header"].get("msg_id"):
# wrap SIGINT handler
real_handler = signal.getsignal(signal.SIGINT)
def double_int(sig, frame):
# call real handler (forwards sigint to kernel),
# then raise local interrupt, stopping local raw_input
real_handler(sig, frame)
raise KeyboardInterrupt
signal.signal(signal.SIGINT, double_int)
content = req['content']
read = getpass if content.get('password', False) else input
try:
raw_data = read(content["prompt"])
except EOFError:
# turn EOFError into EOF character
raw_data = '\x04'
except KeyboardInterrupt:
sys.stdout.write('\n')
return
finally:
# restore SIGINT handler
signal.signal(signal.SIGINT, real_handler)
# only send stdin reply if there *was not* another request
# or execution finished while we were reading.
if not (run_sync(self.client.stdin_channel.msg_ready)()
or run_sync(self.client.shell_channel.msg_ready)()):
self.client.input(raw_data)
def complete_request(self, code, cursor_pos):
# send completion request to kernel
# Give the kernel up to 5s to respond
msg_id = self.client.complete(
code=code,
cursor_pos=cursor_pos,
)
msg = run_sync(self.client.shell_channel.get_msg)(timeout=self.timeout)
if msg['parent_header']['msg_id'] == msg_id:
return msg['content']
return {
'matches': [],
'cursor_start': 0,
'cursor_end': 0,
'metadata': {},
'status': 'ok'
}
def handle_is_complete_reply(self, msg_id, timeout=None):
"""
Wait for a repsonse from the kernel, and return two values:
more? - (boolean) should the frontend ask for more input
indent - an indent string to prefix the input
Overloaded methods may want to examine the comeplete source. Its is
in the self._source_lines_buffered list.
"""
## Get the is_complete response:
msg = None
try:
kwargs = {"timeout": timeout}
msg = run_sync(self.client.shell_channel.get_msg)(**kwargs)
except Empty:
warn('The kernel did not respond to an is_complete_request. '
'Setting `use_kernel_is_complete` to False.')
self.use_kernel_is_complete = False
return False, ""
## Handle response:
if msg["parent_header"].get("msg_id", None) != msg_id:
warn(
'The kernel did not respond properly to an is_complete_request: %s.'
% str(msg))
return False, ""
else:
status = msg["content"].get("status", None)
indent = msg["content"].get("indent", "")
## Return more? and indent string
if status == "complete":
return False, indent
elif status == "incomplete":
return True, indent
elif status == "invalid":
raise SyntaxError()
elif status == "unknown":
return False, indent
else:
warn('The kernel sent an invalid is_complete_reply status: "%s".' %
status)
return False, indent
class SpyderKernelManager(QtKernelManager):
"""
Spyder kernels that live in a conda environment are now properly activated
with custom activation scripts located at plugins/ipythonconsole/scripts.
However, on windows the batch script is terminated but not the kernel it
started so this subclass overrides the `_kill_kernel` method to properly
kill the started kernels by using psutil.
"""
def __init__(self, *args, **kwargs):
self.shutting_down = False
return QtKernelManager.__init__(self, *args, **kwargs)
@staticmethod
def kill_proc_tree(pid, sig=signal.SIGTERM, include_parent=True,
timeout=None, on_terminate=None):
"""
Kill a process tree (including grandchildren) with sig and return a
(gone, still_alive) tuple.
"on_terminate", if specified, is a callabck function which is called
as soon as a child terminates.
This is an new method not present in QtKernelManager.
"""
assert pid != os.getpid() # Won't kill myself!
# This is necessary to avoid showing an error when restarting the
# kernel after it failed to start in the first place.
# Fixes spyder-ide/spyder#11872
try:
parent = psutil.Process(pid)
except psutil.NoSuchProcess:
return ([], [])
children = parent.children(recursive=True)
if include_parent:
children.append(parent)
for child_process in children:
# This is necessary to avoid an error when restarting the
# kernel that started a PyQt5 application in the background.
# Fixes spyder-ide/spyder#13999
try:
child_process.send_signal(sig)
except psutil.AccessDenied:
return ([], [])
gone, alive = psutil.wait_procs(
children,
timeout=timeout,
callback=on_terminate,
)
return (gone, alive)
def _kill_kernel_5(self):
"""
Kill the running kernel.
Override private method of jupyter_client 5 to be able to correctly
close kernel that was started via a batch/bash script for correct conda
env activation.
"""
if self.has_kernel:
# Signal the kernel to terminate (sends SIGKILL on Unix and calls
# TerminateProcess() on Win32).
try:
if hasattr(signal, 'SIGKILL'):
self.signal_kernel(signal.SIGKILL)
else:
# This is the additional line that was added to properly
# kill the kernel started by Spyder.
self.kill_proc_tree(self.kernel.pid)
self.kernel.kill()
except OSError as e:
# In Windows, we will get an Access Denied error if the process
# has already terminated. Ignore it.
if sys.platform == 'win32':
if e.winerror != 5:
raise
# On Unix, we may get an ESRCH error if the process has already
# terminated. Ignore it.
else:
from errno import ESRCH
if e.errno != ESRCH:
raise
# Block until the kernel terminates.
self.kernel.wait()
self.kernel = None
else:
raise RuntimeError("Cannot kill kernel. No kernel is running!")
async def _async_kill_kernel(self, restart: bool = False) -> None:
"""Kill the running kernel.
Override private method of jupyter_client 7 to be able to correctly
close kernel that was started via a batch/bash script for correct conda
env activation.
"""
if self.has_kernel:
assert self.provisioner is not None
# This is the additional line that was added to properly
# kill the kernel started by Spyder.
self.kill_proc_tree(self.provisioner.process.pid)
await self.provisioner.kill(restart=restart)
# Wait until the kernel terminates.
import asyncio
try:
await asyncio.wait_for(self._async_wait(), timeout=5.0)
except asyncio.TimeoutError:
# Wait timed out, just log warning but continue
# - not much more we can do.
self.log.warning("Wait for final termination of kernel timed"
" out - continuing...")
pass
else:
# Process is no longer alive, wait and clear
if self.has_kernel:
await self.provisioner.wait()
# override alias for jupyter_client < 7
if JUPYTER_CLIENT_GE_7:
_kill_kernel = run_sync(_async_kill_kernel)
else:
_kill_kernel = _kill_kernel_5
def handle_iopub(self, msg_id=''):
"""Process messages on the IOPub channel
This method consumes and processes messages on the IOPub channel,
such as stdout, stderr, execute_result and status.
It only displays output that is caused by this session.
"""
while run_sync(self.client.iopub_channel.msg_ready)():
sub_msg = run_sync(self.client.iopub_channel.get_msg)()
msg_type = sub_msg['header']['msg_type']
# Update execution_count in case it changed in another session
if msg_type == "execute_input":
self.execution_count = int(
sub_msg["content"]["execution_count"]) + 1
if self.include_output(sub_msg):
if msg_type == 'status':
self._execution_state = sub_msg["content"][
"execution_state"]
elif msg_type == 'stream':
if sub_msg["content"]["name"] == "stdout":
if self._pending_clearoutput:
print("\r", end="")
self._pending_clearoutput = False
print(sub_msg["content"]["text"], end="")
sys.stdout.flush()
elif sub_msg["content"]["name"] == "stderr":
if self._pending_clearoutput:
print("\r", file=sys.stderr, end="")
self._pending_clearoutput = False
print(sub_msg["content"]["text"],
file=sys.stderr,
end="")
sys.stderr.flush()
elif msg_type == 'execute_result':
if self._pending_clearoutput:
print("\r", end="")
self._pending_clearoutput = False
self.execution_count = int(
sub_msg["content"]["execution_count"])
if not self.from_here(sub_msg):
sys.stdout.write(self.other_output_prefix)
format_dict = sub_msg["content"]["data"]
self.handle_rich_data(format_dict)
if 'text/plain' not in format_dict:
continue
# prompt_toolkit writes the prompt at a slightly lower level,
# so flush streams first to ensure correct ordering.
sys.stdout.flush()
sys.stderr.flush()
self.print_out_prompt()
text_repr = format_dict['text/plain']
if '\n' in text_repr:
# For multi-line results, start a new line after prompt
print()
print(text_repr)
# Remote: add new prompt
if not self.from_here(sub_msg):
sys.stdout.write('\n')
sys.stdout.flush()
self.print_remote_prompt()
elif msg_type == 'display_data':
data = sub_msg["content"]["data"]
handled = self.handle_rich_data(data)
if not handled:
if not self.from_here(sub_msg):
sys.stdout.write(self.other_output_prefix)
# if it was an image, we handled it by now
if 'text/plain' in data:
print(data['text/plain'])
# If execute input: print it
elif msg_type == 'execute_input':
content = sub_msg['content']
ec = content.get('execution_count',
self.execution_count - 1)
# New line
sys.stdout.write('\n')
sys.stdout.flush()
# With `Remote In [3]: `
self.print_remote_prompt(ec=ec)
# And the code
sys.stdout.write(content['code'] + '\n')
elif msg_type == 'clear_output':
if sub_msg["content"]["wait"]:
self._pending_clearoutput = True
else:
print("\r", end="")
elif msg_type == 'error':
for frame in sub_msg["content"]["traceback"]:
print(frame, file=sys.stderr)
class SpyderKernelManager(QtKernelManager):
"""
Spyder kernels that live in a conda environment are now properly activated
with custom activation scripts located at plugins/ipythonconsole/scripts.
However, on windows the batch script is terminated but not the kernel it
started so this subclass overrides the `_kill_kernel` method to properly
kill the started kernels by using psutil.
"""
client_class = DottedObjectName(
'spyder.plugins.ipythonconsole.utils.client.SpyderKernelClient')
def __init__(self, *args, **kwargs):
self.shutting_down = False
return QtKernelManager.__init__(self, *args, **kwargs)
@staticmethod
async def kill_proc_tree(pid,
sig=signal.SIGTERM,
include_parent=True,
timeout=None,
on_terminate=None):
"""
Kill a process tree (including grandchildren) with sig and return a
(gone, still_alive) tuple.
"on_terminate", if specified, is a callabck function which is called
as soon as a child terminates.
This is an new method not present in QtKernelManager.
"""
assert pid != os.getpid() # Won't kill myself!
# This is necessary to avoid showing an error when restarting the
# kernel after it failed to start in the first place.
# Fixes spyder-ide/spyder#11872
try:
parent = psutil.Process(pid)
except psutil.NoSuchProcess:
return ([], [])
children = parent.children(recursive=True)
if include_parent:
children.append(parent)
for child_process in children:
# This is necessary to avoid an error when restarting the
# kernel that started a PyQt5 application in the background.
# Fixes spyder-ide/spyder#13999
try:
child_process.send_signal(sig)
except psutil.AccessDenied:
return ([], [])
gone, alive = psutil.wait_procs(
children,
timeout=timeout,
callback=on_terminate,
)
return (gone, alive)
async def _async_kill_kernel(self, restart: bool = False) -> None:
"""Kill the running kernel.
Override private method of jupyter_client 7 to be able to correctly
close kernel that was started via a batch/bash script for correct conda
env activation.
"""
if self.has_kernel:
assert self.provisioner is not None
# This is the additional line that was added to properly
# kill the kernel started by Spyder.
await self.kill_proc_tree(self.provisioner.process.pid)
await self.provisioner.kill(restart=restart)
# Wait until the kernel terminates.
import asyncio
try:
await asyncio.wait_for(self._async_wait(), timeout=5.0)
except asyncio.TimeoutError:
# Wait timed out, just log warning but continue
# - not much more we can do.
self.log.warning("Wait for final termination of kernel timed"
" out - continuing...")
pass
else:
# Process is no longer alive, wait and clear
if self.has_kernel:
await self.provisioner.wait()
_kill_kernel = run_sync(_async_kill_kernel)
async def _async_send_kernel_sigterm(self, restart: bool = False) -> None:
"""similar to _kill_kernel, but with sigterm (not sigkill), but do not block"""
if self.has_kernel:
assert self.provisioner is not None
# This is the line that was added to properly kill kernels started
# by Spyder.
await self.kill_proc_tree(self.provisioner.process.pid)
_send_kernel_sigterm = run_sync(_async_send_kernel_sigterm)
def subkernel_download(self, line):
"""Download a file/directory from a subkernel's file system."""
args = parse_argstring(self.subkernel_download, line)
run_sync(self.download_handler)(self.binding_manager.get(args.name),
args.remote_path, args.local_path)
