def _run_signaltest_lifecycle(self, config=None):
km = KernelManager(config=config, kernel_name='signaltest')
kc = self._prepare_kernel(km, stdout=PIPE, stderr=PIPE)
def execute(cmd):
request_id = kc.execute(cmd)
while True:
reply = kc.get_shell_msg(TIMEOUT)
if reply['parent_header']['msg_id'] == request_id:
break
content = reply['content']
assert content['status'] == 'ok'
return content
execute("start")
assert km.is_alive()
execute('check')
assert km.is_alive()
km.restart_kernel(now=True)
assert km.is_alive()
execute('check')
km.shutdown_kernel()
assert km.context.closed
def _run_signaltest_lifecycle(self, config=None):
km = KernelManager(config=config, kernel_name="signaltest")
kc = self._prepare_kernel(km, stdout=PIPE, stderr=PIPE)
def execute(cmd):
request_id = kc.execute(cmd)
while True:
reply = kc.get_shell_msg(TIMEOUT)
if reply["parent_header"]["msg_id"] == request_id:
break
content = reply["content"]
assert content["status"] == "ok"
return content
execute("start")
assert km.is_alive()
execute("check")
assert km.is_alive()
km.restart_kernel(now=True)
assert km.is_alive()
execute("check")
km.shutdown_kernel()
assert km.context.closed
class ThreadedExecutor(object):
def __init__(self, name, queue=Queue()):
self.name = name
self.comms = Comms(self)
self.km = None
self.kc = None
self.execution_count = 0
Channel.queue = queue
self.start()
def start(self):
self.km = KernelManager(kernel_name='python',
client_class='execute.Client')
self.km.start_kernel()
self.kc = self.km.client()
self.kc.start_channels()
time.sleep(2)
atexit.register(self.shutdown_kernel)
def restart_kernel(self):
self.execution_count = 0
self.km.restart_kernel()
Channel.executions = OrderedDict()
def interrupt_kernel(self):
self.km.interrupt_kernel()
def shutdown_kernel(self): # TODO: Shutdown kernel but keep labmode running
self.execution_count = 0
self.km.request_shutdown()
self.km.cleanup()
self.km.finish_shutdown()
logging.info('Shutting down')
def kernel_info(self):
self.kc.kernel_info()
def __call__(self, code, stop_on_error=True, cell=None, silent=False):
"If stop_on_error is True, execution may stop on exceptions"
if not silent:
self.execution_count += 1
if cell:
cell.prompt = self.execution_count
self.kc.execute(code,
silent=silent,
store_history=True, # Has to be true to make execution counter work
allow_stdin=False,
stop_on_error=stop_on_error)
class ThreadedExecutor(object):
def __init__(self, name, queue=Queue()):
self.name = name
self.comms = Comms(self)
self.km = None
self.kc = None
self.execution_count = 0
self.executable = None
Channel.queue = queue
def start(self, cwd=None, executable=None):
self.km = KernelManager(kernel_name='python',
client_class='nei.execute.Client')
self.executable = executable
if executable is not None:
self.km.kernel_spec.argv[0] = executable
self.km.start_kernel(**({} if cwd is None else {'cwd': cwd}))
self.kc = self.km.client()
self.kc.start_channels()
time.sleep(2)
atexit.register(self.shutdown_kernel)
def reset(self):
Channel.executions = 0
def restart_kernel(self):
self.execution_count = 0
self.km.restart_kernel()
Channel.executions = 0
def interrupt_kernel(self):
self.km.interrupt_kernel()
def shutdown_kernel(self): # TODO: Shutdown kernel but keep nei running
self.execution_count = 0
self.km.request_shutdown()
self.km.cleanup()
self.km.finish_shutdown()
logging.info('Shutting down')
def kernel_info(self):
self.kc.kernel_info()
def complete(self, code, position):
self.kc.complete(code, position)
def __call__(self, code, stop_on_error=True, cell=None, silent=False):
"If stop_on_error is True, execution may stop on exceptions"
if not silent:
self.execution_count += 1
if cell:
cell.prompt = self.execution_count
self.kc.execute(
code,
silent=silent,
store_history=True, # Has to be true to make execution counter work
allow_stdin=False,
stop_on_error=stop_on_error)
class JupyterClient:
_logger = _module_logger.getChild('JupyterClient')
TIMEOUT = None # -1 ???
##############################################
def __init__(self,
working_directory,
kernel='python3',
embed_kernel=False):
if embed_kernel:
self._kernel_manager = InProcessKernelManager(kernel_name=kernel)
else:
self._kernel_manager = KernelManager(kernel_name=kernel)
stderr = open(os.devnull, 'w')
self._kernel_manager.start_kernel(cwd=working_directory, stderr=stderr)
self._init_client()
##############################################
def __del__(self):
self.close()
##############################################
def _init_client(self):
self._kernel_client = self._kernel_manager.client()
self._kernel_client.start_channels()
try:
self._kernel_client.wait_for_ready()
except RuntimeError:
message = 'Timeout from starting kernel'
self._logger.error(message)
# \nTry restarting python session and running again
self._kernel_client.stop_channels()
self._kernel_manager.shutdown_kernel()
raise TimeoutError(message)
self._kernel_client.allow_stdin = False
##############################################
def close(self):
self._logger.info('Stop kernel')
# Pweave as this line
# Fixme: block ??? not documented ???
# self._kernel_client.stop_channels()
self._kernel_manager.shutdown_kernel(now=True)
##############################################
def restart(self):
self._logger.info('Restart kernel')
# self._kernel_client.shutdown(restart=True) # ???
# self._kernel_client.stop_channels()
self._kernel_manager.restart_kernel(now=False) # 1s to cleanup
# do we have to sleep ???
self._init_client()
##############################################
@staticmethod
def message_id_match(message, message_id):
return message['parent_header'].get('msg_id') == message_id
##############################################
def _wait_for_finish(self, message_id):
"""Wait for finish, with timeout"""
# self._logger.debug('wait for finish, with timeout')
while True:
try:
# Get a message from the shell channel
# timeout = self.timeout
# if timeout < 0:
# timeout = None
message = self._kernel_client.get_shell_msg(
timeout=self.TIMEOUT)
# self._logger.debug('message {}'.format(message))
except Empty:
# if self._interrupt_on_timeout:
# self._kernel_manager.interrupt_kernel()
# break
message = 'Cell execution timed out' # , see log for details.
self._logger.error(message)
raise TimeoutError(message)
if self.message_id_match(message, message_id):
break
else:
# not our reply
continue
# self._logger.debug('wait for finish done')
##############################################
def run_cell(self, source):
"""Execute the source.
Return a list of :class:`nbformat.NotebookNode` instance.
"""
source = source.lstrip()
cell = {}
cell['source'] = source
message_id = self._kernel_client.execute(source, store_history=False)
# self._logger.debug('message_id {}'.format(message_id))
self._wait_for_finish(message_id)
outputs = JupyterOutputs()
while True:
try:
# We've already waited for execute_reply, so all output should already be
# waiting. However, on slow networks, like in certain CI systems, waiting < 1 second
# might miss messages. So long as the kernel sends a status:idle message when it
# finishes, we won't actually have to wait this long, anyway.
message = self._kernel_client.iopub_channel.get_msg(block=True,
timeout=4)
# self._logger.debug('message {}'.format(message))
except Empty:
message = 'Timeout waiting for IOPub output'
self._logger.error(message)
# \nTry restarting python session and running again
raise TimeoutError(message)
# stdout from InProcessKernelManager has no parent_header
if not self.message_id_match(
message, message_id) and message['msg_type'] != 'stream':
continue
message_type = message['msg_type']
content = message['content']
# self._logger.debug('msg_type {}'.format(message_type))
# self._logger.debug('content {}'.format(content))
# set the prompt number for the input and the output
if 'execution_count' in content:
cell['execution_count'] = content['execution_count']
if message_type == 'status':
if content['execution_state'] == 'idle':
break # exit while loop
else:
continue
elif message_type == 'execute_input':
continue
elif message_type == 'clear_output':
outputs = JupyterOutputs()
continue
elif message_type.startswith('comm'):
continue
try:
output = nbformat_v4.output_from_msg(message)
except ValueError:
self._logger.error(
'unhandled iopub message: {}'.format(message_type))
else:
outputs.append(JupyterOutput(output))
return outputs
