def __init__(self, *args, **kwargs):
super(MetaKernel, self).__init__(*args, **kwargs)
if MetaKernel.meta_kernel is None:
MetaKernel.meta_kernel = self
if self.log is None:
# This occurs if we call as a stand-alone kernel
# (eg, not as a process)
# FIXME: take care of input/output, eg StringIO
# make work without a session
self.log = logging.Logger(".metakernel")
else:
# Write has already been set
try:
sys.stdout.write = self.Write
except:
pass # Can't change stdout
self.redirect_to_log = False
self.shell = None
self.sticky_magics = OrderedDict()
self._i = None
self._ii = None
self._iii = None
self._ = None
self.__ = None
self.___ = None
self.max_hist_cache = 1000
self.hist_cache = []
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
self.hist_file = get_history_file(self)
self.parser = Parser(self.identifier_regex, self.func_call_regex,
self.magic_prefixes, self.help_suffix)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type)
self._ipy_formatter = IPythonDisplayFormatter()
self.env = {}
self.reload_magics()
# provide a way to get the current instance
self.set_variable("kernel", self)
# Run command line filenames, if given:
if self.parent is not None and self.parent.extra_args:
level = self.log.level
self.log.setLevel("INFO")
self.redirect_to_log = True
self.Write("Executing files...")
for filename in self.parent.extra_args:
self.Write(" %s..." % filename)
try:
self.do_execute_file(filename)
except Exception as exc:
self.log.info(" %s" % (exc,))
self.Write("Executing files: done!")
self.log.setLevel(level)
self.redirect_to_log = False
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._replace_get_ipython()
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.shell_handlers['comm_open'] = self.comm_manager.comm_open
self.shell_handlers['comm_msg'] = self.comm_manager.comm_msg
self.shell_handlers['comm_close'] = self.comm_manager.comm_close
if ipywidgets_extension_loaded:
self.comm_manager.register_target('ipython.widget', Widget.handle_comm_opened)
self._start_polymake()
def __init__(self, **kwargs):
# sp = super(YAPKernel, self)
super(YAPKernel, self).__init__(**kwargs)
# Initialize the InteractiveShell subclass
self.shell = self.shell_class.instance(
parent=self,
profile_dir=self.profile_dir,
user_ns=self.user_ns,
kernel=self,
)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('execute_result')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
self.comm_manager = CommManager(parent=self, kernel=self)
# self.shell._last_traceback = None
self.shell.configurables.append(self.comm_manager)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager,
msg_type)
self.yap_shell = YAPInteractiveShell(self)
def __init__(self, **kwargs):
super(GLFKernel, self).__init__(**kwargs)
# set up the shell
self.shell = self.shell_class.instance(
parent=self,
profile_dir=self.profile_dir,
user_module=self.user_module,
user_ns=self.user_ns,
kernel=self,
)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('execute_result')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
# set up and attach comm_manager to the shell
self.comm_manager = CommManager(parent=self, kernel=self)
self.shell.configurables.append(self.comm_manager)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager,
msg_type)
# initialize the GFRepl
self.GFRepl = GLFRepl(GF_BIN)
def __init__(self, *args, **kwargs):
super(MetaKernel, self).__init__(*args, **kwargs)
if MetaKernel.meta_kernel is None:
MetaKernel.meta_kernel = self
if self.log is None:
# This occurs if we call as a stand-alone kernel
# (eg, not as a process)
# FIXME: take care of input/output, eg StringIO
# make work without a session
self.log = logging.Logger(".metakernel")
else:
# Write has already been set
try:
sys.stdout.write = self.Write
except:
pass # Can't change stdout
self.sticky_magics = {}
self._i = None
self._ii = None
self._iii = None
self._ = None
self.__ = None
self.___ = None
self.max_hist_cache = 1000
self.hist_cache = []
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
self.plot_settings = dict(backend='inline')
self.hist_file = get_history_file(self)
self.reload_magics()
# provide a way to get the current instance
self.set_variable("kernel", self)
self.parser = Parser(self.identifier_regex, self.func_call_regex,
self.magic_prefixes, self.help_suffix)
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type)
self._ipy_formatter = IPythonDisplayFormatter()
self.env = {}
def __init__(self, *args, **kwargs):
super(MetaKernel, self).__init__(*args, **kwargs)
if MetaKernel.meta_kernel is None:
MetaKernel.meta_kernel = self
if self.log is None:
# This occurs if we call as a stand-alone kernel
# (eg, not as a process)
# FIXME: take care of input/output, eg StringIO
# make work without a session
self.log = logging.Logger(".metakernel")
else:
# Write has already been set
try:
sys.stdout.write = self.Write
except:
pass # Can't change stdout
self.redirect_to_log = False
self.shell = None
self.sticky_magics = OrderedDict()
self._i = None
self._ii = None
self._iii = None
self._ = None
self.__ = None
self.___ = None
self.max_hist_cache = 1000
self.hist_cache = []
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
self.hist_file = get_history_file(self)
self.parser = Parser(self.identifier_regex, self.func_call_regex,
self.magic_prefixes, self.help_suffix)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type)
self._ipy_formatter = IPythonDisplayFormatter()
self.env = {}
self.reload_magics()
# provide a way to get the current instance
self.set_variable("kernel", self)
# Run command line filenames, if given:
if self.parent is not None and self.parent.extra_args:
level = self.log.level
self.log.setLevel("INFO")
self.redirect_to_log = True
self.Write("Executing files...")
for filename in self.parent.extra_args:
self.Write(" %s..." % filename)
try:
self.do_execute_file(filename)
except Exception as exc:
self.log.info(" %s" % (exc,))
self.Write("Executing files: done!")
self.log.setLevel(level)
self.redirect_to_log = False
def __init__(self, *args, **kwargs):
super(MetaKernel, self).__init__(*args, **kwargs)
if MetaKernel.meta_kernel is None:
MetaKernel.meta_kernel = self
if self.log is None:
# This occurs if we call as a stand-alone kernel
# (eg, not as a process)
# FIXME: take care of input/output, eg StringIO
# make work without a session
self.log = logging.Logger(".metakernel")
else:
# Write has already been set
try:
sys.stdout.write = self.Write
except:
pass # Can't change stdout
self.sticky_magics = {}
self._i = None
self._ii = None
self._iii = None
self._ = None
self.__ = None
self.___ = None
self.max_hist_cache = 1000
self.hist_cache = []
self.comm_manager = CommManager(shell=None, parent=self, kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
self.plot_settings = dict(backend='inline')
self.hist_file = get_history_file(self)
self.reload_magics()
# provide a way to get the current instance
self.set_variable("kernel", self)
self.parser = Parser(self.identifier_regex, self.func_call_regex,
self.magic_prefixes, self.help_suffix)
self.comm_manager = CommManager(shell=None, parent=self, kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager,
msg_type)
self._ipy_formatter = IPythonDisplayFormatter()
self.env = {}
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._replace_get_ipython()
self.comm_manager = CommManager(shell=None, parent=self, kernel=self)
self.shell_handlers["comm_open"] = self.comm_manager.comm_open
self.shell_handlers["comm_msg"] = self.comm_manager.comm_msg
self.shell_handlers["comm_close"] = self.comm_manager.comm_close
self.comm_manager.register_target("ipython.widget", Widget.handle_comm_opened)
self._start_singular()
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._replace_get_ipython()
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.shell_handlers['comm_open'] = self.comm_manager.comm_open
self.shell_handlers['comm_msg'] = self.comm_manager.comm_msg
self.shell_handlers['comm_close'] = self.comm_manager.comm_close
if ipywidgets_extension_loaded:
self.comm_manager.register_target('ipython.widget', Widget.handle_comm_opened)
self._start_polymake()
def __init__(self):
super(FlaskKernel, self).__init__()
self.session = SessionWebsocket(parent=self, key=SESSION_KEY)
self.stream = self.iopub_socket = WebsocketStream(self.session)
self.iopub_socket.channel = 'iopub'
self.session.stream = self.iopub_socket
self.comm_manager = CommManager(parent=self, kernel=self)
self.shell = None
self.log = logging.getLogger('fake')
comm_msg_types = [ 'comm_open', 'comm_msg', 'comm_close' ]
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type)
def __init__(self, key=None, document=None):
super(BokehKernel, self).__init__()
self.session = PanelSessionWebsocket(document=document, parent=self, key=key)
self.stream = self.iopub_socket = WebsocketStream(self.session)
self.iopub_socket.channel = 'iopub'
self.session.stream = self.iopub_socket
self.comm_manager = CommManager(parent=self, kernel=self)
self.shell = None
self.log = logging.getLogger('fake')
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type)
def create_shell(self):
shell = self.shell_class.instance(
parent=self,
profile_dir=self.profile_dir,
user_module=self.user_module,
user_ns=self.user_ns,
kernel=self,
)
comm_manager = CommManager(shell=shell, parent=self, kernel=self)
shell.configurables.append(comm_manager)
self.shell_registry[id(shell)] = shell
self.comm_manager_registry[id(shell)] = comm_manager
return shell, comm_manager
def establish_comm_manager(self):
# see ipykernel/ipkernel.py
self.shell = self.shell_class.instance(parent=self,
profile_dir=self.profile_dir,
user_module=self.user_module,
user_ns=self.user_ns,
kernel=self)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('execute_result')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
self.comm_manager = CommManager(parent=self, kernel=self)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager,
msg_type)
def __init__(self, **kwargs):
super(DisplayKernel, self).__init__(**kwargs)
# Configure IPython shell
self.shell = self.shell_class.instance(
parent=self,
profile_dir=self.profile_dir,
user_module=self.user_module,
user_ns=self.user_ns,
kernel=self,
)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('execute_result')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
self.comm_manager = CommManager(parent=self, kernel=self)
self.shell.configurables.append(self.comm_manager)
for type_ in ['comm_open', 'comm_msg', 'comm_close']:
self.shell_handlers[type_] = getattr(self.comm_manager, type_)
class MetaKernel(Kernel):
"""The base MetaKernel class."""
app_name = 'metakernel'
identifier_regex = r'[^\d\W][\w\.]*'
func_call_regex = r'([^\d\W][\w\.]*)\([^\)\()]*\Z'
magic_prefixes = dict(magic='%', shell='!', help='?')
help_suffix = '?'
help_links = [
{
'text': "MetaKernel Magics",
'url': "https://metakernel.readthedocs.io/en/latest/source/README.html",
},
]
language_info = {
# 'mimetype': 'text/x-python',
# 'name': 'python',
# ------ If different from 'language':
# 'codemirror_mode': {
# "version": 2,
# "name": "ipython"
# }
# 'pygments_lexer': 'language',
# 'version' : "x.y.z",
# 'file_extension': '.py',
'help_links': help_links,
}
plot_settings = Dict(dict(backend='inline')).tag(config=True)
meta_kernel = None
@classmethod
def run_as_main(cls, *args, **kwargs):
"""Launch or install a metakernel.
Modules implementing a metakernel subclass can use the following lines:
if __name__ == '__main__':
MetaKernelSubclass.run_as_main()
"""
kwargs['app_name'] = cls.app_name
MetaKernelApp.launch_instance(kernel_class=cls, *args, **kwargs)
def __init__(self, *args, **kwargs):
super(MetaKernel, self).__init__(*args, **kwargs)
if MetaKernel.meta_kernel is None:
MetaKernel.meta_kernel = self
if self.log is None:
# This occurs if we call as a stand-alone kernel
# (eg, not as a process)
# FIXME: take care of input/output, eg StringIO
# make work without a session
self.log = logging.Logger(".metakernel")
else:
# Write has already been set
try:
sys.stdout.write = self.Write
except:
pass # Can't change stdout
self.redirect_to_log = False
self.shell = None
self.sticky_magics = OrderedDict()
self._i = None
self._ii = None
self._iii = None
self._ = None
self.__ = None
self.___ = None
self.max_hist_cache = 1000
self.hist_cache = []
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
self.hist_file = get_history_file(self)
self.parser = Parser(self.identifier_regex, self.func_call_regex,
self.magic_prefixes, self.help_suffix)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type)
self._ipy_formatter = IPythonDisplayFormatter()
self.env = {}
self.reload_magics()
# provide a way to get the current instance
self.set_variable("kernel", self)
# Run command line filenames, if given:
if self.parent is not None and self.parent.extra_args:
level = self.log.level
self.log.setLevel("INFO")
self.redirect_to_log = True
self.Write("Executing files...")
for filename in self.parent.extra_args:
self.Write(" %s..." % filename)
try:
self.do_execute_file(filename)
except Exception as exc:
self.log.info(" %s" % (exc,))
self.Write("Executing files: done!")
self.log.setLevel(level)
self.redirect_to_log = False
def makeSubkernel(self, kernel):
"""
Run this method in an IPython kernel to set
this kernel's input/output settings.
"""
from IPython import get_ipython
from IPython.display import display
shell = get_ipython()
if shell: # we are running under an IPython kernel
self.session = shell.kernel.session
self.Display = display
self.send_response = self._send_shell_response
else:
self.session = kernel.session
self.send_response = kernel.send_response
self.Display = kernel.Display
#####################################
# Methods which provide kernel - specific behavior
def set_variable(self, name, value):
"""
Set a variable to a Python-typed value.
"""
pass
def get_variable(self, name):
"""
Lookup a variable name and return a Python-typed value.
"""
pass
def repr(self, item):
"""The repr of the kernel."""
return repr(item)
def get_usage(self):
"""Get the usage statement for the kernel."""
return "This is a usage statement."
def get_kernel_help_on(self, info, level=0, none_on_fail=False):
"""Get help on an object. Called by the help magic."""
if none_on_fail:
return None
else:
return "Sorry, no help is available on '%s'." % info['code']
def handle_plot_settings(self):
"""Handle the current plot settings"""
pass
def get_local_magics_dir(self):
"""
Returns the path to local magics dir (eg ~/.ipython/metakernel/magics)
"""
base = get_ipython_dir()
return os.path.join(base, 'metakernel', 'magics')
def get_completions(self, info):
"""
Get completions from kernel based on info dict.
"""
return []
def do_execute_direct(self, code, silent=False):
"""
Execute code in the kernel language.
"""
pass
def do_execute_file(self, filename):
"""
Default code for running a file. Just opens the file, and sends
the text to do_execute_direct.
"""
code = "".join(open(filename).readlines())
return self.do_execute_direct(code)
def do_execute_meta(self, code):
"""
Execute meta code in the kernel. This uses the execute infrastructure
but allows JavaScript to talk directly to the kernel bypassing normal
processing.
When responding to the %%debug magic, the step and reset meta
commands can answer with a string in the format:
"highlight: [start_line, start_col, end_line, end_col]"
for highlighting expressions in the frontend.
"""
if code == "reset":
raise Exception("This kernel does not implement this meta command")
elif code == "stop":
raise Exception("This kernel does not implement this meta command")
elif code == "step":
raise Exception("This kernel does not implement this meta command")
elif code.startswith("inspect "):
raise Exception("This kernel does not implement this meta command")
else:
raise Exception("Unknown meta command: '%s'" % code)
def initialize_debug(self, code):
"""
This function is used with the %%debug magic for highlighting
lines of code, and for initializing debug functions.
Return the empty string if highlighting is not supported.
"""
#return "highlight: [%s, %s, %s, %s]" % (line1, col1, line2, col2)
return ""
def do_function_direct(self, function_name, arg):
"""
Call a function in the kernel language with args (as a single item).
"""
f = self.do_execute_direct(function_name)
return f(arg)
def restart_kernel(self):
"""Restart the kernel"""
pass
############################################
# Implement base class methods
def do_execute(self, code, silent=False, store_history=True, user_expressions=None,
allow_stdin=False):
"""Handle code execution.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#execute
"""
# Set the ability for the kernel to get standard-in:
self._allow_stdin = allow_stdin
# Create a default response:
self.kernel_resp = {
'status': 'ok',
# The base class increments the execution count
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
}
# TODO: remove this when IPython fixes this
# This happens at startup when the language is set to python
if '_usage.page_guiref' in code:
return self.kernel_resp
if code and store_history:
self.hist_cache.append(code.strip())
if not code.strip():
return self.kernel_resp
info = self.parse_code(code)
self.payload = []
retval = None
if info['magic'] and info['magic']['name'] == 'help':
if info['magic']['type'] == 'line':
level = 0
else:
level = 1
text = self.get_help_on(code, level)
if text:
content = {
"start_line_number": 0,
"source": "page",
}
if isinstance(text, dict):
content["data"] = text ## {mime-type: ..., mime-type:...}
self.log.debug(str(text))
else:
content["data"] = {"text/plain": text}
self.log.debug(text)
self.payload = [content]
elif info['magic'] or self.sticky_magics:
retval = None
if self.sticky_magics:
magics, code = _split_magics_code(code, self.magic_prefixes)
code = magics + self._get_sticky_magics() + code
stack = []
# Handle magics:
magic = None
prefixes = ((self.magic_prefixes['shell'],
self.magic_prefixes['magic']))
while code.startswith(prefixes):
magic = self.get_magic(code)
if magic is not None:
stack.append(magic)
code = magic.get_code()
# signal to exit, maybe error or no block
if not magic.evaluate:
break
else:
break
# Execute code, if any:
if ((magic is None or magic.evaluate) and code.strip() != ""):
if code.startswith("~~META~~:"):
retval = self.do_execute_meta(code[9:].strip())
else:
retval = self.do_execute_direct(code)
# Post-process magics:
for magic in reversed(stack):
retval = magic.post_process(retval)
else:
if code.startswith("~~META~~:"):
retval = self.do_execute_meta(code[9:].strip())
else:
retval = self.do_execute_direct(code)
self.post_execute(retval, code, silent)
if 'payload' in self.kernel_resp:
self.kernel_resp['payload'] = self.payload
return self.kernel_resp
def post_execute(self, retval, code, silent):
"""Post-execution actions
Handle special kernel variables and display response if not silent.
"""
# Handle in's
self.set_variable("_iii", self._iii)
self.set_variable("_ii", self._ii)
self.set_variable("_i", code)
self.set_variable("_i" + str(self.execution_count), code)
self._iii = self._ii
self._ii = code
if (retval is not None):
# --------------------------------------
# Handle out's (only when non-null)
self.set_variable("___", self.___)
self.set_variable("__", self.__)
self.set_variable("_", retval)
self.set_variable("_" + str(self.execution_count), retval)
self.___ = self.__
self.__ = retval
self.log.debug(retval)
if isinstance(retval, ExceptionWrapper):
self.kernel_resp['status'] = 'error'
content = {
'traceback': retval.traceback,
'evalue': retval.evalue,
'ename': retval.ename,
}
self.kernel_resp.update(content)
if not silent:
self.send_response(self.iopub_socket, 'error', content)
else:
try:
data = _formatter(retval, self.repr)
except Exception as e:
self.Error(e)
return
content = {
'execution_count': self.execution_count,
'data': data[0],
'metadata': data[1],
}
if not silent:
if Widget and isinstance(retval, Widget):
self.Display(retval)
return
self.send_response(self.iopub_socket, 'execute_result', content)
def do_history(self, hist_access_type, output, raw, session=None,
start=None, stop=None, n=None, pattern=None, unique=False):
"""
Access history at startup.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#history
"""
with open(self.hist_file) as fid:
self.hist_cache = json.loads(fid.read() or "[]")
return {'history': [(None, None, h) for h in self.hist_cache]}
def do_shutdown(self, restart):
"""
Shut down the app gracefully, saving history.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#kernel-shutdown
"""
if self.hist_file:
with open(self.hist_file, "w") as fid:
json.dump(self.hist_cache[-self.max_hist_cache:], fid)
if restart:
self.Print("Restarting kernel...")
self.restart_kernel()
self.reload_magics()
self.Print("Done!")
return {'status': 'ok', 'restart': restart}
def do_is_complete(self, code):
"""
Given code as string, returns dictionary with 'status' representing
whether code is ready to evaluate. Possible values for status are:
'complete' - ready to evaluate
'incomplete' - not yet ready
'invalid' - invalid code
'unknown' - unknown; the default unless overridden
Optionally, if 'status' is 'incomplete', you may indicate
an indentation string.
Example:
return {'status' : 'incomplete',
'indent': ' ' * 4}
https://jupyter-client.readthedocs.io/en/stable/messaging.html#code-completeness
"""
if code.startswith(self.magic_prefixes['magic']):
## force requirement to end with an empty line
if code.endswith("\n"):
return {'status' : 'complete'}
else:
return {'status' : 'incomplete'}
# otherwise, how to know is complete?
elif code.endswith("\n"):
return {'status' : 'complete'}
else:
return {'status' : 'incomplete'}
def do_complete(self, code, cursor_pos):
"""Handle code completion for the kernel.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#completion
"""
info = self.parse_code(code, 0, cursor_pos)
content = {
'matches': [],
'cursor_start': info['start'],
'cursor_end': info['end'],
'status': 'ok'
}
matches = info['path_matches']
if info['magic']:
# if the last line contains another magic, use that
line_info = self.parse_code(info['line'])
if line_info['magic']:
info = line_info
if info['magic']['type'] == 'line':
magics = self.line_magics
else:
magics = self.cell_magics
if info['magic']['name'] in magics:
magic = magics[info['magic']['name']]
info = info['magic']
if info['type'] == 'cell' and info['code']:
info = self.parse_code(info['code'])
else:
info = self.parse_code(info['args'])
matches.extend(magic.get_completions(info))
elif not info['magic']['code'] and not info['magic']['args']:
matches = []
for name in magics.keys():
if name.startswith(info['magic']['name']):
pre = info['magic']['prefix']
matches.append(pre + name)
info['start'] -= len(pre)
info['full_obj'] = pre + info['full_obj']
info['obj'] = pre + info['obj']
else:
matches.extend(self.get_completions(info))
if info['full_obj'] and len(info['full_obj']) > len(info['obj']):
new_list = [m for m in matches if m.startswith(info['full_obj'])]
if new_list:
content['cursor_end'] = (content['cursor_end'] +
len(info['full_obj']) -
len(info['obj']))
matches = new_list
content["matches"] = sorted(matches)
return content
def do_inspect(self, code, cursor_pos, detail_level=0):
"""Object introspection.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#introspection
"""
if cursor_pos > len(code):
return
content = {'status': 'aborted', 'data': {}, 'found': False}
docstring = self.get_help_on(code, detail_level, none_on_fail=True,
cursor_pos=cursor_pos)
if docstring:
content["status"] = "ok"
content["found"] = True
if isinstance(docstring, dict): ## {"text/plain": ..., mime-type: ...}
content["data"] = docstring
self.log.debug(str(docstring))
else:
content["data"] = {"text/plain": docstring}
self.log.debug(docstring)
return content
def clear_output(self, wait=False):
"""Clear the output of the kernel."""
self.send_response(self.iopub_socket, 'clear_output',
{'wait': wait})
def Display(self, *objects, **kwargs):
"""Display one or more objects using rich display.
Supports a `clear_output` keyword argument that clears the output before displaying.
See https://ipython.readthedocs.io/en/stable/config/integrating.html?highlight=display#rich-display
"""
if kwargs.get('clear_output'):
self.clear_output(wait=True)
for item in objects:
if Widget and isinstance(item, Widget):
self.log.debug('Display Widget')
self._ipy_formatter(item)
else:
self.log.debug('Display Data')
try:
data = _formatter(item, self.repr)
except Exception as e:
self.Error(e)
return
content = {
'data': data[0],
'metadata': data[1]
}
self.send_response(
self.iopub_socket,
'display_data',
content
)
def Print(self, *objects, **kwargs):
"""Print `objects` to the iopub stream, separated by `sep` and followed by `end`.
Items can be strings or `Widget` instances.
"""
for item in objects:
if Widget and isinstance(item, Widget):
self.Display(item)
objects = [i for i in objects if not (Widget and isinstance(i, Widget))]
message = format_message(*objects, **kwargs)
stream_content = {
'name': 'stdout', 'text': message}
self.log.debug('Print: %s' % message.rstrip())
if self.redirect_to_log:
self.log.info(message.rstrip())
else:
self.send_response(self.iopub_socket, 'stream', stream_content)
def Write(self, message):
"""Write message directly to the iopub stdout with no added end character."""
stream_content = {
'name': 'stdout', 'text': message}
self.log.debug('Write: %s' % message)
if self.redirect_to_log:
self.log.info(message)
else:
self.send_response(self.iopub_socket, 'stream', stream_content)
def Error(self, *objects, **kwargs):
"""Print `objects` to stdout, separated by `sep` and followed by `end`.
Objects are cast to strings.
"""
message = format_message(*objects, **kwargs)
self.log.debug('Error: %s' % message.rstrip())
stream_content = {
'name': 'stderr',
'text': RED + message + NORMAL
}
if self.redirect_to_log:
self.log.info(message.rstrip())
else:
self.send_response(self.iopub_socket, 'stream', stream_content)
##############################
# Private API and methods not likely to be overridden
def reload_magics(self):
"""Reload all of the line and cell magics."""
self.line_magics = {}
self.cell_magics = {}
# get base magic files and those relative to the current class
# directory
magic_files = []
# Make a metakernel/magics if it doesn't exist:
local_magics_dir = get_local_magics_dir()
# Search all of the places there could be magics:
try:
paths = [os.path.join(os.path.dirname(
os.path.abspath(inspect.getfile(self.__class__))), "magics")]
except:
paths = []
paths += [local_magics_dir,
os.path.join(os.path.dirname(os.path.abspath(__file__)), "magics")]
for magic_dir in paths:
sys.path.append(magic_dir)
magic_files.extend(glob.glob(os.path.join(magic_dir, "*.py")))
for magic in magic_files:
basename = os.path.basename(magic)
if basename == "__init__.py":
continue
try:
module = __import__(os.path.splitext(basename)[0])
imp.reload(module)
module.register_magics(self)
except Exception as e:
self.log.error("Can't load '%s': error: %s" % (magic, e))
def register_magics(self, magic_klass):
"""Register magics for a given magic_klass."""
magic = magic_klass(self)
line_magics = magic.get_magics('line')
cell_magics = magic.get_magics('cell')
for name in line_magics:
self.line_magics[name] = magic
for name in cell_magics:
self.cell_magics[name] = magic
def send_response(self, *args, **kwargs):
### if we are running via %parallel, we might not have a
### session
if self.session:
super(MetaKernel, self).send_response(*args, **kwargs)
def call_magic(self, line):
"""
Given an line, such as "%download http://example.com/", parse
and execute magic.
"""
return self.get_magic(line)
def get_magic(self, text):
## FIXME: Bad name, use call_magic instead.
# if first line matches a magic,
# call magic.call_magic() and return magic object
info = self.parse_code(text)
magic = self.line_magics['magic']
return magic.get_magic(info)
def get_magic_args(self, text):
# if first line matches a magic,
# call magic.call_magic() and return magic args
info = self.parse_code(text)
magic = self.line_magics['magic']
return magic.get_magic(info, get_args=True)
def get_help_on(self, expr, level=0, none_on_fail=False,
cursor_pos=-1):
"""Get help for an expression using the help magic."""
help_magic = self.line_magics['help']
return help_magic.get_help_on(expr, level, none_on_fail, cursor_pos)
def parse_code(self, code, cursor_start=0, cursor_end=-1):
"""Parse code using our parser."""
return self.parser.parse_code(code, cursor_start, cursor_end)
def _get_sticky_magics(self):
retval = ""
for key in self.sticky_magics:
retval += (key + " " + self.sticky_magics[key] + "\n")
return retval
def _send_shell_response(self, socket, stream_type, content):
publish_display_data({ 'text/plain': content['text'] })
class polymakeKernel(Kernel):
implementation = 'jupyter_polymake_wrapper'
implementation_version = __version__
help_links = [ { 'text': "Polymake website", 'url': "http://polymake.org/" },
{ 'text': "Polymake documentation", 'url': "https://polymake.org/doku.php/documentation" },
{ 'text': "Polymake tutorial", 'url': "https://polymake.org/doku.php/tutorial/start" },
{ 'text': "Polymake reference", 'url': "https://polymake.org/release_docs/3.0/" } ]
def _replace_get_ipython(self):
new_kernel = own_ipython(self)
global kernel_object_for_ipython
kernel_object_for_ipython = new_kernel
@property
def language_version(self):
m = version_pat.search(self.banner)
return m.group(1)
_banner = None
@property
def banner(self):
if self._banner is None:
self._banner = "Polymake Jupyter kernel"
return self._banner
language_info = {'name': 'polymake',
'codemirror_mode': 'perl', #
'mimetype': 'text/x-polymake',
'file_extension': '.pm'} # FIXME: Is this even real?
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._replace_get_ipython()
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.shell_handlers['comm_open'] = self.comm_manager.comm_open
self.shell_handlers['comm_msg'] = self.comm_manager.comm_msg
self.shell_handlers['comm_close'] = self.comm_manager.comm_close
if ipywidgets_extension_loaded:
self.comm_manager.register_target('ipython.widget', Widget.handle_comm_opened)
self._start_polymake()
def _start_polymake(self):
sig = signal.signal(signal.SIGINT, signal.SIG_DFL)
try:
polymake_run_command = pexpect.which( "polymake" )
self.polymakewrapper = pexpect.spawnu( polymake_run_command + " -" )
# set jupyter enviroment in polymake
try:
self._run_polymake_command( 'prefer "threejs";' )
self._run_polymake_command( 'include "common::jupyter.rules";' )
self._run_polymake_command( '$common::is_used_in_jupyter = 1;' )
except PolymakeRunException:
return False
finally:
signal.signal(signal.SIGINT, sig)
def _run_polymake_command( self, code ):
self.polymakewrapper.sendline( code.rstrip() + '; ' + 'print "===endofoutput===";' )
self.polymakewrapper.expect( 'print "===endofoutput===";' )
error_number = self.polymakewrapper.expect( [ "ERROR", "===endofoutput===" ] )
output = self.polymakewrapper.before.strip().rstrip()
if error_number == 0:
self.polymakewrapper.sendline( 'print "===endofoutput===";' )
self.polymakewrapper.expect( 'print "===endofoutput===";' )
output = 'Error' + self.polymakewrapper.before
self.polymakewrapper.expect( "===endofoutput===" )
raise PolymakeRunException( output )
return output
def _process_python( self, code ):
if code.find( "@python" ) == -1 and code.find( "@widget" ) == -1:
return False
exec(code[7:],globals(),locals())
return True
def do_execute(self, code, silent, store_history=True,
user_expressions=None, allow_stdin=False):
default_return = {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
if not code.strip():
return default_return
if self._process_python( code ):
return default_return
interrupted = False
code = code.rstrip()
#stream_content = {'execution_count': self.execution_count, 'data': { 'text/plain': "Code:\n" + code_to_execute } }
#self.send_response( self.iopub_socket, 'execute_result', stream_content )
try:
output = self._run_polymake_command( code )
except KeyboardInterrupt:
self.polymakewrapper.child.sendintr()
self._run_polymake_command( '' )
interrupted = True
except pexpect.EOF:
output = self.polymakewrapper.before + 'Restarting polymake'
self._start_polymake()
except PolymakeRunException as exception:
output = exception.args[0]
return {'status': 'error', 'execution_count': self.execution_count,
'ename': 'PolymakeRunException', 'evalue': output, 'traceback': []}
if not silent:
while output.find( '.@@HTML@@' ) != -1:
html_position = output.find( '.@@HTML@@' )
html_end_position = output.find( '.@@ENDHTML@@' )
if html_position > 0:
before_html = output[:html_position-1].rstrip()
else:
before_html = ''
output_html = output[html_position+9:html_end_position-1].strip().rstrip()
output = output[html_end_position+12:].strip()
if before_html != '':
stream_content = {'execution_count': self.execution_count, 'data': { 'text/plain': before_html } }
self.send_response( self.iopub_socket, 'execute_result', stream_content )
stream_content = {'execution_count': self.execution_count,
'source' : "polymake",
'data': { 'text/html': output_html},
'metadata': dict() }
self.send_response( self.iopub_socket, 'display_data', stream_content )
if len(output) != 0:
stream_content = {'execution_count': self.execution_count, 'data': { 'text/plain': output } }
self.send_response( self.iopub_socket, 'execute_result', stream_content )
if interrupted:
return {'status': 'abort', 'execution_count': self.execution_count}
return {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
def do_shutdown(self, restart):
self.polymakewrapper.terminate(force=True)
if restart:
self._start_polymake()
### basic code completion for polymake
### currently known shortcomings: intermediate completion, in particular for files, completion of variable names
def code_completion (self,code):
completion = []
code = re.sub( "\)$", "", code)
code = repr(code)
code_line = 'print Jupyter::tab_completion(' + code + ');'
try:
output = self._run_polymake_command( code_line )
except PolymakeRunException:
return (0,[])
completion = output.split("###")
if ( len(completion) > 1 ) :
completion_length = completion.pop(0)
else :
completion_length = 0
return (completion_length,completion)
def do_complete(self, code, cursor_pos):
completion_length, completion = self.code_completion(code[0:cursor_pos])
cur_start = cursor_pos - int(completion_length)
return {'matches': completion, 'cursor_start': cur_start,
'cursor_end': cursor_pos, 'metadata': dict(),
'status': 'ok'}
def do_is_complete( self, code ):
new_code = 'if(0){ ' + code + ' }'
try:
self._run_polymake_command( new_code )
except PolymakeRunException:
return {'status' : 'incomplete', 'indent': '' }
return {'status' : 'complete' }
def do_inspect( self, code, cursor_pos, detail_level=0 ):
new_code = 'print Jupyter::context_help( q#' + code + '#, 1, "text" );'
try:
output = self._run_polymake_command( new_code )
except PolymakeRunException:
return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False}
if output == '':
return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False}
else:
return {'status': 'ok', 'data': { 'text/plain': output }, 'metadata': {}, 'found': True}
class SingularKernel(Kernel):
implementation = "jupyter_singular_wrapper"
implementation_version = __version__
def _replace_get_ipython(self):
new_kernel = own_ipython(self)
global kernel_object_for_ipython
kernel_object_for_ipython = new_kernel
@property
def language_version(self):
m = version_pat.search(self.banner)
return m.group(1)
_banner = None
@property
def banner(self):
if self._banner is None:
self._banner = "Singular Jupyter kernel"
return self._banner
language_info = {
"name": "Singular",
"codemirror_mode": "singular", # note that this does not exist yet
"mimetype": "text/x-singular",
"file_extension": ".singular",
}
help_links = [
{"text": "Singular manual", "url": "http://www.singular.uni-kl.de/Manual/latest/index.htm"},
{"text": "Singular examples", "url": "http://www.singular.uni-kl.de/Manual/latest/sing_842.htm"},
{"text": "Singular library", "url": "http://www.singular.uni-kl.de/Manual/latest/sing_926.htm"},
{"text": "Singular index", "url": "http://www.singular.uni-kl.de/Manual/latest/sing_2336.htm"},
]
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._replace_get_ipython()
self.comm_manager = CommManager(shell=None, parent=self, kernel=self)
self.shell_handlers["comm_open"] = self.comm_manager.comm_open
self.shell_handlers["comm_msg"] = self.comm_manager.comm_msg
self.shell_handlers["comm_close"] = self.comm_manager.comm_close
self.comm_manager.register_target("ipython.widget", Widget.handle_comm_opened)
self._start_singular()
def _start_singular(self):
sig = signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGINT, sig)
InitializeSingular(which("Singular"))
def _check_for_plot(self, code):
return "plot_jupyter" in code
def _process_python(self, code):
if code.find("@python") == -1 and code.find("@widget") == -1:
return False
exec(code[7:], globals(), locals())
return True
def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False):
default = {"status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}}
if not code.strip():
return default
if self._process_python(code):
return default
interrupted = False
code_stripped = code.rstrip()
output = RunSingularCommand(code_stripped)
output_error = output[0]
output_string = output[1]
if not output_error:
if not silent:
if self._check_for_plot(code_stripped):
filename_image = output_string.rstrip()
with open(filename_image, "rb") as imageFile:
image_string = base64.b64encode(imageFile.read()).decode()
stream_content = {
"source": "singular",
"data": {"image/jpeg": image_string},
"metadata": {"image/jpeg": {"width": 400, "height": 400}},
}
self.send_response(self.iopub_socket, "display_data", stream_content)
elif output_string.strip() != "":
stream_content = {"execution_count": self.execution_count, "data": {"text/plain": output_string}}
self.send_response(self.iopub_socket, "execute_result", stream_content)
return {"status": "ok", "execution_count": self.execution_count, "payload": [], "user_expressions": {}}
else:
stream_content = {"execution_count": self.execution_count, "data": {"text/plain": "Error:"}}
self.send_response(self.iopub_socket, "execute_result", stream_content)
stream_content = {"execution_count": self.execution_count, "data": {"text/plain": output_string}}
self.send_response(self.iopub_socket, "execute_result", stream_content)
stream_content = {
"execution_count": self.execution_count,
"ename": "",
"evalue": output_string,
"traceback": [],
}
self.send_response(self.iopub_socket, "error", stream_content)
return {
"status": "error",
"execution_count": self.execution_count,
"ename": "",
"evalue": output_string,
"traceback": [],
}
def do_complete(self, code, cursor_pos):
code = code[:cursor_pos]
default = {"matches": [], "cursor_start": 0, "cursor_end": cursor_pos, "metadata": dict(), "status": "ok"}
token = code.encode("utf-8")
start = cursor_pos - len(token)
completion_list = GetSingularCompletion(code, start, cursor_pos)
if not completion_list:
return default
return {
"matches": sorted(completion_list),
"cursor_start": start,
"cursor_end": cursor_pos,
"metadata": dict(),
"status": "ok",
}
def do_is_complete(self, code):
code = code.rstrip()
if code[-1] == ";":
return {"status": "complete"}
else:
return {"status": "incomplete", "indent": " "}
class MetaKernel(Kernel):
identifier_regex = r'[^\d\W][\w\.]*'
func_call_regex = r'([^\d\W][\w\.]*)\([^\)\()]*\Z'
magic_prefixes = dict(magic='%', shell='!', help='?')
help_suffix = '?'
help_links = [
{
'text': "MetaKernel Magics",
'url': "https://github.com/calysto/metakernel/blob/master/metakernel/magics/README.md",
},
]
language_info = {
# 'mimetype': 'text/x-python',
# 'name': 'python',
# ------ If different from 'language':
# 'codemirror_mode': {
# "version": 2,
# "name": "ipython"
# }
# 'pygments_lexer': 'language',
# 'version' : "x.y.z",
# 'file_extension': '.py',
'help_links': help_links,
}
meta_kernel = None
def __init__(self, *args, **kwargs):
super(MetaKernel, self).__init__(*args, **kwargs)
if MetaKernel.meta_kernel is None:
MetaKernel.meta_kernel = self
if self.log is None:
# This occurs if we call as a stand-alone kernel
# (eg, not as a process)
# FIXME: take care of input/output, eg StringIO
# make work without a session
self.log = logging.Logger(".metakernel")
else:
# Write has already been set
try:
sys.stdout.write = self.Write
except:
pass # Can't change stdout
self.sticky_magics = {}
self._i = None
self._ii = None
self._iii = None
self._ = None
self.__ = None
self.___ = None
self.max_hist_cache = 1000
self.hist_cache = []
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
self.plot_settings = dict(backend='inline')
self.hist_file = get_history_file(self)
self.reload_magics()
# provide a way to get the current instance
self.set_variable("kernel", self)
self.parser = Parser(self.identifier_regex, self.func_call_regex,
self.magic_prefixes, self.help_suffix)
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type)
self._ipy_formatter = IPythonDisplayFormatter()
self.env = {}
@classmethod
def subkernel(cls, kernel):
"""
Handle issues regarding making a kernel a subkernel to this
one. Used in %parallel and %kernel.
"""
pass
def makeSubkernelTo(self, main_kernel, display_function):
"""
Handle issues regarding making this kernel be a subkernel to
another. Used in making metakernels be magics for IPython.
"""
self.session = main_kernel.session
self.Display = display_function
def makeSubkernelToIPython(self):
"""
Run this method in an IPython kernel to set
this kernel's input/output settings.
"""
from IPython import get_ipython
from IPython.display import display
ip = get_ipython()
if ip: # we are running under an IPython kernel
self.makeSubkernelTo(ip.parent, display)
else:
raise Exception("Need to run under an IPython kernel")
#####################################
# Methods which provide kernel - specific behavior
def set_variable(self, name, value):
"""
Set a variable to a Python-typed value.
"""
pass
def get_variable(self, name):
"""
Lookup a variable name and return a Python-typed value.
"""
pass
def repr(self, item):
return repr(item)
def get_usage(self):
return "This is a usage statement."
def get_kernel_help_on(self, info, level=0, none_on_fail=False):
if none_on_fail:
return None
else:
return "Sorry, no help is available on '%s'." % info['code']
def handle_plot_settings(self):
"""Handle the current plot settings"""
pass
def get_local_magics_dir(self):
"""
Returns the path to local magics dir (eg ~/.ipython/metakernel/magics)
"""
base = get_ipython_dir()
return os.path.join(base, 'metakernel', 'magics')
def get_completions(self, info):
"""
Get completions from kernel based on info dict.
"""
return []
def do_execute_direct(self, code, silent=False):
"""
Execute code in the kernel language.
"""
pass
def do_execute_file(self, filename):
"""
Default code for running a file. Just opens the file, and sends
the text to do_execute_direct.
"""
code = "".join(open(filename).readlines())
return self.do_execute_direct(code)
def do_execute_meta(self, code):
"""
Execute meta code in the kernel. This uses the execute infrastructure
but allows JavaScript to talk directly to the kernel bypassing normal
processing.
When responding to the %%debug magic, the step and reset meta
commands can answer with a string in the format:
"highlight: [start_line, start_col, end_line, end_col]"
for highlighting expressions in the frontend.
"""
if code == "reset":
raise Exception("This kernel does not implement this meta command")
elif code == "stop":
raise Exception("This kernel does not implement this meta command")
elif code == "step":
raise Exception("This kernel does not implement this meta command")
elif code.startswith("inspect "):
raise Exception("This kernel does not implement this meta command")
else:
raise Exception("Unknown meta command: '%s'" % code)
def initialize_debug(self, code):
"""
This function is used with the %%debug magic for highlighting
lines of code, and for initializing debug functions.
Return the empty string if highlighting is not supported.
"""
#return "highlight: [%s, %s, %s, %s]" % (line1, col1, line2, col2)
return ""
def do_function_direct(self, function_name, arg):
"""
Call a function in the kernel language with args (as a single item).
"""
self.Error("This language does not support \"%pmap function args\".")
def restart_kernel(self):
"""Restart the kernel"""
pass
############################################
# Implement base class methods
def do_execute(self, code, silent=False, store_history=True, user_expressions=None,
allow_stdin=False):
# Set the ability for the kernel to get standard-in:
self._allow_stdin = allow_stdin
# Create a default response:
self.kernel_resp = {
'status': 'ok',
# The base class increments the execution count
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
}
# TODO: remove this when IPython fixes this
# This happens at startup when the language is set to python
if '_usage.page_guiref' in code:
return self.kernel_resp
if code and store_history:
self.hist_cache.append(code.strip())
if not code.strip():
return self.kernel_resp
info = self.parse_code(code)
self.payload = []
retval = None
if info['magic'] and info['magic']['name'] == 'help':
if info['magic']['type'] == 'line':
level = 0
else:
level = 1
text = self.get_help_on(code, level)
self.log.debug(text)
if text:
self.payload = [{"data": {"text/plain": text},
"start_line_number": 0,
"source": "page"}]
elif info['magic'] or self.sticky_magics:
retval = None
if self.sticky_magics:
magics, code = _split_magics_code(code, self.magic_prefixes)
code = magics + self._get_sticky_magics() + code
stack = []
# Handle magics:
magic = None
prefixes = ((self.magic_prefixes['shell'],
self.magic_prefixes['magic']))
while code.startswith(prefixes):
magic = self.get_magic(code)
if magic is not None:
stack.append(magic)
code = magic.get_code()
# signal to exit, maybe error or no block
if not magic.evaluate:
break
else:
break
# Execute code, if any:
if ((magic is None or magic.evaluate) and code.strip() != ""):
if code.startswith("~~META~~:"):
retval = self.do_execute_meta(code[9:].strip())
else:
retval = self.do_execute_direct(code)
# Post-process magics:
for magic in reversed(stack):
retval = magic.post_process(retval)
else:
if code.startswith("~~META~~:"):
retval = self.do_execute_meta(code[9:].strip())
else:
retval = self.do_execute_direct(code)
self.post_execute(retval, code, silent)
if 'payload' in self.kernel_resp:
self.kernel_resp['payload'] = self.payload
return self.kernel_resp
def post_execute(self, retval, code, silent):
# Handle in's
self.set_variable("_iii", self._iii)
self.set_variable("_ii", self._ii)
self.set_variable("_i", code)
self.set_variable("_i" + str(self.execution_count), code)
self._iii = self._ii
self._ii = code
if (retval is not None):
# --------------------------------------
# Handle out's (only when non-null)
self.set_variable("___", self.___)
self.set_variable("__", self.__)
self.set_variable("_", retval)
self.set_variable("_" + str(self.execution_count), retval)
self.___ = self.__
self.__ = retval
self.log.debug(retval)
try:
content = {'execution_count': self.execution_count,
'data': _formatter(retval, self.repr),
'metadata': dict()}
except Exception as e:
self.Error(e)
return
if not silent:
if Widget and isinstance(retval, Widget):
self.Display(retval)
return
self.send_response(self.iopub_socket, 'execute_result', content)
def do_history(self, hist_access_type, output, raw, session=None,
start=None, stop=None, n=None, pattern=None, unique=False):
"""
Access history at startup.
"""
if not self.hist_file:
return {'history': []}
# else:
if not os.path.exists(self.hist_file):
with open(self.hist_file, 'wb') as fid:
fid.write('')
with open(self.hist_file, 'rb') as fid:
history = fid.read().decode('utf-8', 'replace')
history = history.splitlines()
history = history[:self.max_hist_cache]
self.hist_cache = history
history = [(None, None, h) for h in history]
return {'history': history}
def do_shutdown(self, restart):
"""
Shut down the app gracefully, saving history.
"""
if self.hist_file:
with open(self.hist_file, 'wb') as fid:
data = '\n'.join(self.hist_cache[-self.max_hist_cache:])
fid.write(data.encode('utf-8'))
if restart:
self.Print("Restarting kernel...")
self.reload_magics()
self.restart_kernel()
self.Print("Done!")
return {'status': 'ok', 'restart': restart}
def do_is_complete(self, code):
"""
Given code as string, returns dictionary with 'status' representing
whether code is ready to evaluate. Possible values for status are:
'complete' - ready to evaluate
'incomplete' - not yet ready
'invalid' - invalid code
'unknown' - unknown; the default unless overridden
Optionally, if 'status' is 'incomplete', you may indicate
an indentation string.
Example:
return {'status' : 'incomplete',
'indent': ' ' * 4}
"""
return {'status' : 'unknown'}
def do_complete(self, code, cursor_pos):
info = self.parse_code(code, 0, cursor_pos)
content = {
'matches': [],
'cursor_start': info['start'],
'cursor_end': info['end'],
'metadata': {},
'status': 'ok'
}
matches = info['path_matches']
if info['magic']:
# if the last line contains another magic, use that
line_info = self.parse_code(info['line'])
if line_info['magic']:
info = line_info
if info['magic']['type'] == 'line':
magics = self.line_magics
else:
magics = self.cell_magics
if info['magic']['name'] in magics:
magic = magics[info['magic']['name']]
info = info['magic']
if info['type'] == 'cell' and info['code']:
info = self.parse_code(info['code'])
else:
info = self.parse_code(info['args'])
matches.extend(magic.get_completions(info))
elif not info['magic']['code'] and not info['magic']['args']:
matches = []
for name in magics.keys():
if name.startswith(info['magic']['name']):
pre = info['magic']['prefix']
matches.append(pre + name)
info['start'] -= len(pre)
info['full_obj'] = pre + info['full_obj']
info['obj'] = pre + info['obj']
else:
matches.extend(self.get_completions(info))
if info['full_obj'] and len(info['full_obj']) > len(info['obj']):
new_list = [m for m in matches if m.startswith(info['full_obj'])]
if new_list:
content['cursor_end'] = (content['cursor_end'] +
len(info['full_obj']) -
len(info['obj']))
matches = new_list
content["matches"] = sorted(matches)
return content
def do_inspect(self, code, cursor_pos, detail_level=0):
# Object introspection
if cursor_pos > len(code):
return
content = {'status': 'aborted', 'data': {}, 'found': False}
docstring = self.get_help_on(code, detail_level, none_on_fail=True,
cursor_pos=cursor_pos)
if docstring:
content["data"] = {"text/plain": docstring}
content["status"] = "ok"
content["found"] = True
self.log.debug(docstring)
return content
##############################
# Private API and methods not likely to be overridden
def reload_magics(self):
self.line_magics = {}
self.cell_magics = {}
# get base magic files and those relative to the current class
# directory
magic_files = []
# Make a metakernel/magics if it doesn't exist:
local_magics_dir = get_local_magics_dir()
# Search all of the places there could be magics:
paths = [local_magics_dir,
os.path.join(os.path.dirname(os.path.abspath(__file__)), "magics")]
try:
paths += [os.path.join(os.path.dirname(
os.path.abspath(inspect.getfile(self.__class__))), "magics")]
except:
pass
for magic_dir in paths:
sys.path.append(magic_dir)
magic_files.extend(glob.glob(os.path.join(magic_dir, "*.py")))
for magic in magic_files:
basename = os.path.basename(magic)
if basename == "__init__.py":
continue
try:
module = __import__(os.path.splitext(basename)[0])
imp.reload(module)
module.register_magics(self)
except Exception as e:
self.log.error("Can't load '%s': error: %s" % (magic, e))
def register_magics(self, magic_klass):
magic = magic_klass(self)
line_magics = magic.get_magics('line')
cell_magics = magic.get_magics('cell')
for name in line_magics:
self.line_magics[name] = magic
for name in cell_magics:
self.cell_magics[name] = magic
def clear_output(self, wait=False):
self.send_response(self.iopub_socket, 'clear_output',
{'wait': wait})
def Display(self, *args, **kwargs):
clear_output = kwargs.get("clear_output", False)
for message in args:
if isinstance(message, HTML):
if clear_output:
self.send_response(self.iopub_socket, 'clear_output',
{'wait': True})
if Widget and isinstance(message, Widget):
self.log.debug('Display Widget')
self._ipy_formatter(message)
else:
self.log.debug('Display Data')
try:
data = _formatter(message, self.repr)
except Exception as e:
self.Error(e)
return
self.send_response(self.iopub_socket, 'display_data',
{'data': data,
'metadata': dict()})
def Print(self, *args, **kwargs):
end = kwargs["end"] if ("end" in kwargs) else "\n"
message = ""
for item in args:
if Widget and isinstance(item, Widget):
self.Display(item)
else:
if message:
message += " "
if PY3:
message += str(item)
else:
message += codecs.encode(item, "utf-8")
message += end
stream_content = {
'name': 'stdout', 'text': message, 'metadata': dict()}
self.log.debug('Print: %s' % message)
self.send_response(self.iopub_socket, 'stream', stream_content)
def Write(self, message):
stream_content = {
'name': 'stdout', 'text': message, 'metadata': dict()}
self.log.debug('Write: %s' % message)
self.send_response(self.iopub_socket, 'stream', stream_content)
def Error(self, *args, **kwargs):
message = format_message(*args, **kwargs)
self.log.debug('Error: %s' % message)
stream_content = {
'name': 'stderr', 'text': message, 'metadata': dict()}
self.send_response(self.iopub_socket, 'stream', stream_content)
def call_magic(self, line):
"""
Given an line, such as "%download http://example.com/", parse
and execute magic.
"""
return self.get_magic(line)
def get_magic(self, text):
## FIXME: Bad name, use call_magic instead.
# if first line matches a magic,
# call magic.call_magic() and return magic object
info = self.parse_code(text)
magic = self.line_magics['magic']
return magic.get_magic(info)
def get_magic_args(self, text):
# if first line matches a magic,
# call magic.call_magic() and return magic args
info = self.parse_code(text)
magic = self.line_magics['magic']
return magic.get_magic(info, get_args=True)
def get_help_on(self, expr, level=0, none_on_fail=False,
cursor_pos=-1):
help_magic = self.line_magics['help']
return help_magic.get_help_on(expr, level, none_on_fail, cursor_pos)
def parse_code(self, code, cursor_start=0, cursor_end=-1):
return self.parser.parse_code(code, cursor_start, cursor_end)
def _get_sticky_magics(self):
retval = ""
for key in self.sticky_magics:
retval += (key + " " +
" ".join(self.sticky_magics[key])).strip() + "\n"
return retval
def __init__(self, **kwargs):
start_time = time.time()
Kernel.__init__(self, **kwargs)
logger.debug("session %s %s", type(self.session), self.session)
logger.debug("iopub_socket %s %s", type(self.iopub_socket),
self.iopub_socket)
self.original_iopub_socket = self.iopub_socket
self.iopub_socket = Splitter([self.original_iopub_socket, self])
self.shell = self.shell_class.instance(parent=self,
profile_dir=self.profile_dir,
user_ns=self.user_ns,
kernel=self)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('execute_result')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
self.comm_manager = CommManager(parent=self, kernel=self)
self.shell.configurables.append(self.comm_manager)
self.shell_handlers['comm_open'] = self.comm_open
self.shell_handlers['comm_msg'] = self.comm_msg
self.shell_handlers['comm_close'] = self.comm_close
self.ansible_cfg = None
self.ansible_process = None
self.current_play = None
self.next_task_file = None
self.task_files = []
self.registered_variable = None
self.playbook_file = None
self.silent = False
self.runner = None
self.runner_thread = None
self.shutdown_requested = False
self.shutdown = False
self.widgets = defaultdict(dict)
self.widget_update_order = 0
self.vault_password = None
self.default_inventory = "[all]\nlocalhost ansible_connection=local\n"
self.default_play = yaml.dump(
dict(hosts='localhost', name='default', gather_facts=False))
self.temp_dir = tempfile.mkdtemp(prefix="ansible_kernel_playbook")
self.queue = None
self.tasks_counter = 0
self.current_task = None
logger.debug(self.temp_dir)
os.mkdir(os.path.join(self.temp_dir, 'env'))
os.mkdir(os.path.join(self.temp_dir, 'project'))
os.mkdir(os.path.join(self.temp_dir, 'project', 'roles'))
with open(os.path.join(self.temp_dir, 'env', 'settings'), 'w') as f:
f.write(json.dumps(dict(idle_timeout=0, job_timeout=0)))
self.do_inventory(self.default_inventory)
self.shell.run_code("import json")
self.do_execute_play(self.default_play)
logger.info("Kernel init finished took %s", time.time() - start_time)
class AnsibleKernel(Kernel):
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
shell_class = Type(ZMQInteractiveShell)
implementation = 'ansible_kernel'
implementation_version = __version__
@property
def language_version(self):
m = version_pat.search(self.banner)
return m.group(1)
_banner = None
@property
def banner(self):
if self._banner is None:
self._banner = check_output(['ansible',
'--version']).decode('utf-8')
return self._banner
language_info = {
'name': 'ansible',
'codemirror_mode': 'yaml',
'mimetype': 'text/yaml',
'file_extension': '.yml'
}
help_links = [{
'text': 'Ansible Reference',
'url': 'https://docs.ansible.com/ansible/latest/index.html'
}]
def __init__(self, **kwargs):
start_time = time.time()
Kernel.__init__(self, **kwargs)
logger.debug("session %s %s", type(self.session), self.session)
logger.debug("iopub_socket %s %s", type(self.iopub_socket),
self.iopub_socket)
self.original_iopub_socket = self.iopub_socket
self.iopub_socket = Splitter([self.original_iopub_socket, self])
self.shell = self.shell_class.instance(parent=self,
profile_dir=self.profile_dir,
user_ns=self.user_ns,
kernel=self)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('execute_result')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
self.comm_manager = CommManager(parent=self, kernel=self)
self.shell.configurables.append(self.comm_manager)
self.shell_handlers['comm_open'] = self.comm_open
self.shell_handlers['comm_msg'] = self.comm_msg
self.shell_handlers['comm_close'] = self.comm_close
self.ansible_cfg = None
self.ansible_process = None
self.current_play = None
self.next_task_file = None
self.task_files = []
self.registered_variable = None
self.playbook_file = None
self.silent = False
self.runner = None
self.runner_thread = None
self.shutdown_requested = False
self.shutdown = False
self.widgets = defaultdict(dict)
self.widget_update_order = 0
self.vault_password = None
self.default_inventory = "[all]\nlocalhost ansible_connection=local\n"
self.default_play = yaml.dump(
dict(hosts='localhost', name='default', gather_facts=False))
self.temp_dir = tempfile.mkdtemp(prefix="ansible_kernel_playbook")
self.queue = None
self.tasks_counter = 0
self.current_task = None
logger.debug(self.temp_dir)
os.mkdir(os.path.join(self.temp_dir, 'env'))
os.mkdir(os.path.join(self.temp_dir, 'project'))
os.mkdir(os.path.join(self.temp_dir, 'project', 'roles'))
with open(os.path.join(self.temp_dir, 'env', 'settings'), 'w') as f:
f.write(json.dumps(dict(idle_timeout=0, job_timeout=0)))
self.do_inventory(self.default_inventory)
self.shell.run_code("import json")
self.do_execute_play(self.default_play)
logger.info("Kernel init finished took %s", time.time() - start_time)
def start_helper(self):
self.queue = queue.Queue()
self.helper = AnsibleKernelHelpersThread(self.queue)
self.helper.start()
self.process_widgets()
logger.info("Started helper")
config = configparser.SafeConfigParser()
if self.ansible_cfg is not None:
config.readfp(six.StringIO(self.ansible_cfg))
if not os.path.exists(os.path.join(self.temp_dir, 'project')):
os.mkdir(os.path.join(self.temp_dir, 'project'))
if not config.has_section('defaults'):
config.add_section('defaults')
if config.has_option('defaults', 'roles_path'):
roles_path = config.get('defaults', 'roles_path')
roles_path = ":".join(
[os.path.abspath(x) for x in roles_path.split(":")])
roles_path = "{0}:{1}".format(
roles_path,
os.path.abspath(
pkg_resources.resource_filename('ansible_kernel',
'roles')))
config.set('defaults', 'roles_path', roles_path)
else:
config.set(
'defaults', 'roles_path',
os.path.abspath(
pkg_resources.resource_filename('ansible_kernel',
'roles')))
logger.debug(
"vault_password? %s", self.vault_password
and not config.has_option('defaults', 'vault_password_file'))
if self.vault_password and not config.has_option(
'defaults', 'vault_password_file'):
vault_password_file = os.path.join(self.temp_dir, 'project',
'vault-secret')
with open(vault_password_file, 'w') as vpf:
vpf.write(self.vault_password)
config.set('defaults', 'vault_password_file', vault_password_file)
if not config.has_section('callback_ansible_kernel_helper'):
config.add_section('callback_ansible_kernel_helper')
config.set('callback_ansible_kernel_helper', 'status_port',
str(self.helper.status_socket_port))
with open(os.path.join(self.temp_dir, 'project', 'ansible.cfg'),
'w') as f:
config.write(f)
logger.info("Wrote ansible.cfg")
def rewrite_ports(self):
with open(self.playbook_file, 'r') as f:
playbook = yaml.load(f.read())
playbook[0]['tasks'][0]['pause_for_kernel'][
'port'] = self.helper.pause_socket_port
with open(self.playbook_file, 'w') as f:
f.write(yaml.safe_dump(playbook, default_flow_style=False))
def clean_up_task_files(self, backup=False):
for task_file in self.task_files:
if backup:
shutil.copy(task_file, task_file + ".bak")
if os.path.exists(task_file):
os.unlink(task_file)
self.task_files = []
def runner_process_message(self, data):
logger.info("runner message:\n{}".format(pprint.pformat(data)))
try:
event_data = data.get('event_data', {})
task = event_data.get('task')
role = event_data.get('role', None)
event = data.get('event')
if DEBUG:
stream_content = dict(name='stdout',
text="{}\n".format(pprint.pformat(data)))
self.send_response(self.iopub_socket, 'stream', stream_content)
if event == 'playbook_on_start':
pass
elif event == 'playbook_on_play_start':
pass
elif event == 'playbook_on_stats':
pass
elif event == 'playbook_on_include':
pass
elif event == 'playbook_on_task_start':
logger.debug('playbook_on_task_start')
task_args = event_data.get('task_args', [])
task_uuid = data.get('uuid', '')
self.queue.put(
StatusMessage([
'TaskStart',
dict(task_name=task,
role_name=role,
task_arg=task_args,
task_id=task_uuid)
]))
elif event == 'runner_on_ok':
logger.debug('runner_on_ok')
results = event_data.get('res', {})
device_name = event_data.get('host')
task_uuid = data.get('uuid', '')
self.queue.put(
StatusMessage([
'TaskStatus',
dict(
task_name=task,
role_name=role,
device_name=device_name,
delegated_host_name=device_name,
changed=results.get('changed', False),
failed=False,
unreachable=False,
skipped=False,
application_python=self._format_application_python(
results),
text_html=self._format_text_html(results),
output=self._format_output(results),
error=self._format_error(results),
full_results=json.dumps(results).replace(
'\\', '\\\\'),
results=self._dump_results(results),
task_id=task_uuid)
]))
elif event == 'runner_on_failed':
device_name = event_data.get('host')
task_uuid = data.get('uuid', '')
results = event_data.get('res', {})
self.queue.put(
StatusMessage([
'TaskStatus',
dict(
task_name=task,
role_name=role,
device_name=device_name,
changed=False,
failed=True,
unreachable=False,
skipped=False,
delegated_host_name=device_name,
application_python=self._format_application_python(
results),
text_html=self._format_text_html(results),
output=self._format_output(results),
error=self._format_error(results),
full_results=json.dumps(results).replace(
'\\', '\\\\'),
results=self._dump_results(results),
task_id=task_uuid)
]))
elif event == 'runner_on_unreachable':
device_name = event_data.get('host')
task_uuid = data.get('uuid', '')
self.queue.put(
StatusMessage([
'TaskStatus',
dict(task_name=task,
role_name=role,
device_name=device_name,
changed=False,
failed=False,
unreachable=True,
skipped=False,
task_id=task_uuid)
]))
elif event == 'error':
self.queue.put(
StatusMessage(
['Error', dict(stdout=data.get('stdout', ''))]))
else:
stream_content = dict(name='stdout',
text="{}\n".format(pprint.pformat(data)))
self.send_response(self.iopub_socket, 'stream', stream_content)
except BaseException:
logger.error(traceback.format_exc())
def process_message(self, message):
logger.info("message %s", message)
stop_processing = False
message_type = message[0]
message_data = message[1]
logger.info("message_type %s", message_type)
logger.info("message_data %s", message_data)
if message_data.get('task_name', '') == 'pause_for_kernel':
logger.debug('pause_for_kernel')
return stop_processing
if message_data.get('task_name', '') == 'include_variables':
return stop_processing
if message_data.get('task_name', '') == 'include_vars':
return stop_processing
if message_data.get('task_name', '') == 'include_tasks':
logger.debug('include_tasks')
if message_type == 'TaskStatus' and message_data.get(
'failed', False):
logger.debug('failed')
output = 'fatal: [%s]: FAILED!' % message_data['device_name']
if message_data.get('results', None):
output += " => "
output += message_data['results']
output += "\n"
stream_content = {'name': 'stdout', 'text': str(output)}
self.send_response(self.iopub_socket, 'stream', stream_content)
return stop_processing
output = ''
if message_type == 'TaskStart':
logger.debug('TaskStart')
task_name = message_data['task_name']
if message_data.get('role_name'):
task_name = "%s : %s" % (message_data['role_name'], task_name)
output = 'TASK [%s] %s\n' % (task_name, '*' *
(72 - len(task_name)))
elif message_type == 'DeviceStatus':
logger.debug('DeviceStatus')
pass
elif message_type == 'PlaybookEnded':
logger.debug('PlaybookEnded')
output = "\nPlaybook ended\nContext lost!\n"
self.do_shutdown(False)
self.clean_up_task_files(True)
self.start_helper()
self.rewrite_ports()
self.start_ansible_playbook()
stop_processing = True
elif message_type == 'TaskStatus':
logger.debug('TaskStatus')
if message_data.get('changed', False):
logger.debug('changed')
output = 'changed: [%s]' % message_data['device_name']
elif message_data.get('unreachable', False):
logger.debug('unreachable')
output = 'fatal: [%s]: UNREACHABLE!' % message_data[
'device_name']
elif message_data.get('failed', False):
logger.debug('failed')
output = 'fatal: [%s]: FAILED!' % message_data['device_name']
else:
logger.debug('ok')
output = 'ok: [%s]' % message_data['device_name']
if message_data.get('full_results',
None) and self.registered_variable is not None:
logger.debug('full_results %s',
type(message_data.get('full_results')))
self.shell.run_cell("{0} = json.loads('{1}')".format(
self.registered_variable,
message_data.get('full_results')))
if message_data.get('results', None):
output += " => "
output += message_data['results']
if message_data.get('output', None):
output += "\n\n[%s] stdout:\n" % message_data['device_name']
output += message_data['output']
if message_data.get('error', None):
output += "\n\n[%s] stderr:\n" % message_data['device_name']
output += message_data['error']
if message_data.get('application_python', None):
self.shell.run_cell(message_data.get('application_python'))
if message_data.get('text_html', None):
self.send_response(
self.iopub_socket, 'display_data',
dict(source="",
data={"text/html": message_data.get('text_html')}))
output += "\n"
elif message_type == 'Error':
logger.debug('Error')
output = message_data.get('stdout')
else:
output = str(message)
logger.info("output %s", output)
if not self.silent:
# Send standard output
logger.info("sending output")
stream_content = {'name': 'stdout', 'text': str(output)}
self.send_response(self.iopub_socket, 'stream', stream_content)
else:
logger.info("silent")
logger.info("stop_processing %s", stop_processing)
return stop_processing
def do_execute(self,
code,
silent,
store_history=True,
user_expressions=None,
allow_stdin=False):
self.silent = silent
if not code.strip():
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
logger.debug('code %r', code)
try:
if code.strip().startswith("#inventory"):
return self.do_inventory(code)
elif code.strip().startswith("#ansible.cfg"):
return self.do_ansible_cfg(code)
elif code.strip().startswith("#host_vars"):
return self.do_host_vars(code)
elif code.strip().startswith("#group_vars"):
return self.do_group_vars(code)
elif code.strip().startswith("#vars"):
return self.do_vars(code)
elif code.strip().startswith("#template"):
return self.do_template(code)
elif code.strip().startswith("#task"):
return self.do_execute_task(code)
elif code.strip().startswith("#play"):
return self.do_execute_play(code)
elif code.strip().startswith("#python"):
return self.do_execute_python(code)
elif code.strip().startswith("#vault_password"):
return self.do_execute_vault_password(code)
else:
return self.do_execute_task(code)
except BaseException as e:
logger.error(traceback.format_exc())
reply = {
'status': 'error',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
'traceback': traceback.format_exc().splitlines(),
'ename': type(e).__name__,
'evalue': str(e)
}
self.send_response(self.iopub_socket,
'error',
reply,
ident=self._topic('error'))
return reply
def send_traceback(self, e, limit=None):
reply = {
'status': 'error',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
'traceback': traceback.format_exc(limit).splitlines(),
'ename': type(e).__name__,
'evalue': str(e)
}
self.send_response(self.iopub_socket,
'error',
reply,
ident=self._topic('error'))
return reply
def send_error(self, e, limit=None):
reply = {
'status': 'error',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
'traceback': str(e).splitlines(),
'ename': type(e).__name__,
'evalue': str(e)
}
self.send_response(self.iopub_socket,
'error',
reply,
ident=self._topic('error'))
return reply
def do_inventory(self, code):
logger.info("inventory set to %s", code)
with open(os.path.join(self.temp_dir, 'inventory'), 'w') as f:
f.write("\n".join(code.splitlines()[1:]))
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
def do_ansible_cfg(self, code):
self.ansible_cfg = str(code)
# Test that the code for ansible.cfg is parsable. Do not write the file yet.
try:
config = configparser.SafeConfigParser()
if self.ansible_cfg is not None:
config.readfp(six.StringIO(self.ansible_cfg))
except configparser.ParsingError as e:
return self.send_error(e, 0)
logger.info("ansible.cfg set to %s", code)
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
def do_host_vars(self, code):
code_lines = code.strip().splitlines(True)
host = code_lines[0][len('#host_vars'):].strip()
logger.debug("host %s", host)
host_vars = os.path.join(self.temp_dir, 'project', 'host_vars')
if not os.path.exists(host_vars):
os.mkdir(host_vars)
with open(os.path.join(host_vars, host), 'w') as f:
f.write("".join(code_lines[1:]))
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
def do_vars(self, code):
code_lines = code.strip().splitlines(True)
vars = code_lines[0][len('#vars'):].strip()
logger.debug("vars %s", vars)
with open(os.path.join(self.temp_dir, 'project', vars), 'w') as f:
f.write("".join(code_lines[1:]))
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
def do_template(self, code):
code_lines = code.strip().splitlines(True)
template = code_lines[0][len('#template'):].strip()
logger.debug("template %s", template)
with open(os.path.join(self.temp_dir, 'project', template), 'w') as f:
f.write("".join(code_lines[1:]))
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
def do_group_vars(self, code):
code_lines = code.strip().splitlines(True)
group = code_lines[0][len('#group_vars'):].strip()
logger.debug("group %s", group)
group_vars = os.path.join(self.temp_dir, 'project', 'group_vars')
if not os.path.exists(group_vars):
os.mkdir(group_vars)
with open(os.path.join(group_vars, group), 'w') as f:
f.write("".join(code_lines[1:]))
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
def do_execute_play(self, code):
if self.is_ansible_alive():
self.do_shutdown(False)
self.start_helper()
code_data = yaml.load(code)
logger.debug('code_data %r', code_data)
logger.debug('code_data type: %s', type(code_data))
self.current_play = code
playbook = []
current_play = yaml.load(self.current_play)
if current_play is None:
current_play = {}
playbook.append(current_play)
tasks = current_play['tasks'] = current_play.get('tasks', [])
current_play['roles'] = current_play.get('roles', [])
for role in current_play['roles']:
if "." in role:
self.get_galaxy_role(role)
current_play['roles'].insert(0, 'ansible_kernel_helpers')
tasks.append({
'pause_for_kernel': {
'host': '127.0.0.1',
'port': self.helper.pause_socket_port,
'task_num': self.tasks_counter - 1
}
})
widget_vars_file = os.path.join(self.temp_dir, 'project',
'widget_vars.yml')
with open(widget_vars_file, 'w') as f:
f.write(yaml.dump({}))
tasks.append({'include_vars': {'file': 'widget_vars.yml'}})
tasks.append(
{'include_tasks': 'next_task{0}.yml'.format(self.tasks_counter)})
logger.debug(yaml.safe_dump(playbook, default_flow_style=False))
if not os.path.exists(os.path.join(self.temp_dir, 'project')):
os.mkdir(os.path.join(self.temp_dir, 'project'))
self.playbook_file = (os.path.join(self.temp_dir, 'project',
'playbook.yml'))
with open(self.playbook_file, 'w') as f:
f.write(yaml.safe_dump(playbook, default_flow_style=False))
# Weird work around for streaming content not showing
stream_content = {'name': 'stdout', 'text': '\n'}
self.send_response(self.iopub_socket, 'stream', stream_content)
# End weird work around
self.start_ansible_playbook()
logger.info("done")
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
def start_ansible_playbook(self):
# We may need to purge artifacts when we start again
if os.path.exists(os.path.join(self.temp_dir, 'artifacts')):
shutil.rmtree(os.path.join(self.temp_dir, 'artifacts'))
logger.info("runner starting")
env = os.environ.copy()
env['ANSIBLE_KERNEL_STATUS_PORT'] = str(self.helper.status_socket_port)
self.runner_thread, self.runner = ansible_runner.run_async(
private_data_dir=self.temp_dir,
playbook="playbook.yml",
quiet=True,
debug=True,
ignore_logging=True,
cancel_callback=self.cancel_callback,
finished_callback=self.finished_callback,
event_handler=self.runner_process_message)
logger.info("runner started")
logger.info("Runner status: {}".format(self.runner.status))
while self.runner.status in ['unstarted', 'running', 'starting']:
logger.info("In runner loop")
try:
logger.info("getting message %s",
self.helper.pause_socket_port)
msg = self.queue.get(timeout=1)
except queue.Empty:
logger.info("Queue Empty!")
continue
logger.info(msg)
if isinstance(msg, StatusMessage):
if self.process_message(msg.message):
break
elif isinstance(msg, TaskCompletionMessage):
logger.info('msg.task_num %s tasks_counter %s', msg.task_num,
self.tasks_counter)
break
elif not self.is_ansible_alive():
logger.info("ansible is dead")
self.do_shutdown(False)
break
logger.info("Bottom of runner loop")
time.sleep(1)
logger.info("Runner state is now {}".format(self.runner.status))
self.clean_up_task_files()
logger.info("done")
def process_widgets(self):
# Extract values from widgets
# Values in widgets with a var_name property are added to the vars file
# Values in widgets with a ansible_kernel_property are store into special variables
widget_vars_file = os.path.join(self.temp_dir, 'project',
'widget_vars.yml')
logger.debug("widget_vars_file %s", widget_vars_file)
widget_vars = {}
for widget in sorted(self.widgets.values(),
key=lambda x: x['widget_update_order']):
logger.debug("widget %s", pformat(widget))
if 'var_name' in widget and 'value' in widget:
widget_vars[widget['var_name']] = widget['value']
if 'ansible_kernel_property' in widget and 'value' in widget:
if widget['ansible_kernel_property'] == 'vault_password':
self.vault_password = widget['value']
logger.debug("set vault_password")
# Save the vars from the widgets and include it for this task
with open(widget_vars_file, 'w') as f:
f.write(yaml.safe_dump(widget_vars, default_flow_style=False))
def do_execute_task(self, code):
if not self.is_ansible_alive():
logger.info("ansible is dead")
self.do_shutdown(False)
if self.helper is None:
output = "No play found. Run a valid play cell"
stream_content = {'name': 'stdout', 'text': str(output)}
self.send_response(self.iopub_socket, 'stream', stream_content)
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
self.registered_variable = None
self.current_task = code
try:
code_data = yaml.load(code)
except Exception:
code_data = code
logger.debug('code_data %s', code_data)
logger.debug('code_data type: %s', type(code_data))
if isinstance(code_data, str):
if (code_data.endswith("?")):
module = code_data[:-1].split()[-1]
else:
module = code_data.split()[-1]
data = self.get_module_doc(module)
payload = dict(source='', data=data)
logging.debug('payload %s', payload)
# content = {'name': 'stdout', 'text': str(payload)}
self.send_response(self.iopub_socket, 'display_data', payload)
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
elif isinstance(code_data, list):
code_data = code_data[0]
elif isinstance(code_data, dict):
code_data = code_data
elif code_data is None:
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
else:
logger.error('code_data %s unsupported type', type(code_data))
if not isinstance(code_data, dict):
try:
code_data = yaml.load(code)
tb = []
except Exception:
tb = traceback.format_exc(1).splitlines()
reply = {
'status': 'error',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
'traceback': ['Invalid task cell\n'] + tb,
'ename': 'Invalid cell',
'evalue': ''
}
self.send_response(self.iopub_socket,
'error',
reply,
ident=self._topic('error'))
return reply
if 'include_role' in code_data.keys():
role_name = code_data['include_role'].get('name', '')
if '.' in role_name:
self.get_galaxy_role(role_name)
if 'register' in code_data.keys():
self.registered_variable = code_data['register']
interrupted = False
try:
tasks = []
current_task_data = yaml.load(self.current_task)
current_task_data['ignore_errors'] = True
tasks.append(current_task_data)
tasks.append({
'pause_for_kernel': {
'host': '127.0.0.1',
'port': self.helper.pause_socket_port,
'task_num': self.tasks_counter
}
})
self.process_widgets()
tasks.append({'include_vars': {'file': 'widget_vars.yml'}})
# Create the include file task to look for the future task
tasks.append({
'include_tasks':
'next_task{0}.yml'.format(self.tasks_counter + 1)
})
logger.debug(yaml.safe_dump(tasks, default_flow_style=False))
self.next_task_file = os.path.join(
self.temp_dir, 'project',
'next_task{0}.yml'.format(self.tasks_counter))
self.tasks_counter += 1
self.task_files.append(self.next_task_file)
with open(self.next_task_file, 'w') as f:
f.write(yaml.safe_dump(tasks, default_flow_style=False))
logger.info('Wrote %s', self.next_task_file)
self.helper.pause_socket.send_string('Proceed')
while True:
logger.info("getting message %s",
self.helper.pause_socket_port)
msg = self.queue.get()
logger.info(msg)
if isinstance(msg, StatusMessage):
if self.process_message(msg.message):
break
elif isinstance(msg, TaskCompletionMessage):
logger.info('msg.task_num %s tasks_counter %s',
msg.task_num, self.tasks_counter)
break
except KeyboardInterrupt:
logger.error(traceback.format_exc())
if interrupted:
return {'status': 'abort', 'execution_count': self.execution_count}
return {
'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
def do_execute_python(self, code):
code = "".join(code.splitlines(True)[1:])
reply_content = {}
res = self.shell.run_cell(code)
if res.success:
reply_content['status'] = 'ok'
else:
reply_content['status'] = 'error'
reply_content['execution_count'] = self.execution_count
reply_content['payload'] = self.shell.payload_manager.read_payload()
self.shell.payload_manager.clear_payload()
return reply_content
def do_execute_vault_password(self, code):
self.shell.run_cell(
"import ansible_kernel.widgets\n"
"style = {'description_width': 'initial'}\n"
"ansible_kernel.widgets.VaultPassword(description='Vault Password:'******'status': 'ok',
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {}
}
def do_complete(self, code, cursor_pos):
code = code[:cursor_pos]
default = {
'matches': [],
'cursor_start': 0,
'cursor_end': cursor_pos,
'metadata': dict(),
'status': 'ok'
}
if code.strip().startswith("#inventory"):
return default
elif code.strip().startswith("#ansible.cfg"):
return default
elif code.strip().startswith("#host_vars"):
return default
elif code.strip().startswith("#group_vars"):
return default
elif code.strip().startswith("#task"):
return self.do_complete_task(code, cursor_pos)
elif code.strip().startswith("#play"):
return self.do_complete_play(code, cursor_pos)
else:
return self.do_complete_task(code, cursor_pos)
def do_complete_task(self, code, cursor_pos):
default = {
'matches': [],
'cursor_start': 0,
'cursor_end': cursor_pos,
'metadata': dict(),
'status': 'ok'
}
logger.debug('code %r', code)
if not code or code[-1] == ' ':
return default
found_module = False
code_data = None
try:
code_data = yaml.load(code)
except Exception:
try:
code_data = yaml.load(code + ":")
except Exception:
code_data = None
if code_data is not None:
logger.debug('code_data %s', code_data)
if isinstance(code_data, list) and len(code_data) > 0:
code_data = code_data[0]
if isinstance(code_data, dict):
for key in code_data.keys():
if key in modules:
module_name = key
found_module = True
break
logger.debug('found_module %s', found_module)
tokens = code.split()
if not tokens:
return default
matches = []
token = tokens[-1]
start = cursor_pos - len(token)
logger.debug('token %s', token)
if not found_module:
for module in TASK_ARGS_MODULES:
if module.startswith(token):
matches.append(module)
else:
for arg in module_args.get(module_name, []) + task_args:
if arg.startswith(token):
matches.append(arg)
if not matches:
return default
matches = [m for m in matches if m.startswith(token)]
return {
'matches': sorted(matches),
'cursor_start': start,
'cursor_end': cursor_pos,
'metadata': dict(),
'status': 'ok'
}
def do_complete_play(self, code, cursor_pos):
default = {
'matches': [],
'cursor_start': 0,
'cursor_end': cursor_pos,
'metadata': dict(),
'status': 'ok'
}
logger.debug('code %r', code)
if not code or code[-1] == ' ':
return default
tokens = code.split()
if not tokens:
return default
matches = []
token = tokens[-1]
start = cursor_pos - len(token)
logger.debug('token %s', token)
for arg in play_args:
if arg.startswith(token):
matches.append(arg)
if not matches:
return default
matches = [m for m in matches if m.startswith(token)]
return {
'matches': sorted(matches),
'cursor_start': start,
'cursor_end': cursor_pos,
'metadata': dict(),
'status': 'ok'
}
def do_inspect(self, code, cursor_pos, detail_level=0):
logger.debug("code %s", code)
logger.debug("cursor_pos %s", cursor_pos)
logger.debug("detail_level %s", detail_level)
if code.strip().startswith("#inventory"):
logger.info("#inentory not supported")
return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': True}
elif code.strip().startswith("#task"):
return self.do_inspect_module(code, cursor_pos, detail_level)
elif code.strip().startswith("#play"):
logger.info("#play not supported")
return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': True}
else:
return self.do_inspect_module(code, cursor_pos, detail_level)
def do_inspect_module(self, code, cursor_pos, detail_level=0):
data = dict()
code_data = yaml.load(code)
logger.debug("code_data %s", code_data)
if isinstance(code_data, str):
module = code_data
elif isinstance(code_data, dict):
for arg in task_args:
if arg in code_data:
del code_data[arg]
module = code_data.keys()[0]
else:
logger.warn('code type not supported %s', type(code_data))
return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False}
data.update(self.get_module_doc(module))
return {'status': 'ok', 'data': data, 'metadata': {}, 'found': True}
def get_galaxy_role(self, role_name):
command = ['ansible-galaxy', 'list', '-p', 'project/roles']
logger.debug("command %s", command)
p = Popen(command, cwd=self.temp_dir, stdout=PIPE, stderr=STDOUT)
p.wait()
exitcode = p.returncode
logger.debug('exitcode %s', exitcode)
output = p.communicate()[0].decode('utf-8')
for line in output.splitlines():
if line.startswith('- '):
role, _, version = line[2:].partition(',')
role = role.strip()
if role == role_name:
return
p = Popen(
command,
cwd=self.temp_dir,
stdout=PIPE,
stderr=STDOUT,
)
command = [
'ansible-galaxy', 'install', '-p', 'project/roles', role_name
]
logger.debug("command %s", command)
p = Popen(
command,
cwd=self.temp_dir,
stdout=PIPE,
stderr=STDOUT,
)
p.wait()
exitcode = p.returncode
logger.debug('exitcode %s', exitcode)
output = p.communicate()[0].decode('utf-8')
logger.debug('output %s', output)
stream_content = {'name': 'stdout', 'text': str(output)}
self.send_response(self.iopub_socket, 'stream', stream_content)
def get_module_doc(self, module):
data = {}
logger.debug("command %s",
" ".join(['ansible-doc', '-t', 'module', module]))
p = Popen(
['ansible-doc', '-t', 'module', module],
stdout=PIPE,
stderr=STDOUT,
)
p.wait()
exitcode = p.returncode
logger.debug('exitcode %s', exitcode)
output = p.communicate()[0].decode('utf-8')
logger.debug('output %s', output)
data['text/plain'] = output
return data
def is_ansible_alive(self):
if self.runner_thread is None:
logger.info("NOT STARTED")
return False
if self.runner_thread.is_alive():
logger.info("YES")
else:
logger.info("NO")
return self.runner_thread.is_alive()
def cancel_callback(self):
logger.info('called')
return self.shutdown_requested
def finished_callback(self, runner):
logger.info('called')
self.shutdown = True
if not self.shutdown_requested:
self.queue.put(StatusMessage(['PlaybookEnded', {}]))
def do_shutdown(self, restart):
if self.is_ansible_alive():
self.shutdown = False
self.shutdown_requested = True
while not self.shutdown:
if not self.is_ansible_alive():
break
logger.info("waiting for shutdown")
time.sleep(1)
logger.info("shutdown complete")
self.shutdown_requested = False
self.runner_thread = None
self.runner = None
if self.helper is not None:
self.helper.stop()
self.helper = None
return {'status': 'ok', 'restart': restart}
def _format_application_python(self, result):
if 'application/x-python' in result:
ret_value = result['application/x-python']
del result['application/x-python']
return ret_value
return ""
def _format_text_html(self, result):
if 'text/html' in result:
ret_value = result['text/html']
del result['text/html']
return ret_value
return ""
def _format_output(self, result):
if 'stdout_lines' in result:
return '\n'.join(result['stdout_lines'])
return ""
def _format_error(self, result):
if 'stderr_lines' in result:
return '\n'.join(result['stderr_lines'])
return ""
def _dump_results(self, result):
r = result
for key in [
'_ansible_verbose_always', '_ansible_no_log',
'_ansible_parsed', 'invocation'
]:
if key in r:
del r[key]
if 'stdout' in r:
if r['stdout']:
r['stdout'] = '[see below]'
if 'stdout_lines' in r:
if r['stdout_lines']:
r['stdout_lines'] = '[removed for clarity]'
if 'stderr' in r:
if r['stderr']:
r['stderr'] = '[see below]'
if 'stderr_lines' in r:
if r['stderr_lines']:
r['stderr_lines'] = '[removed for clarity]'
if 'changed' in r:
del r['changed']
if 'reason' in r:
return r['reason']
return json.dumps(r, sort_keys=True, indent=4)
def set_parent(self, ident, parent):
super(AnsibleKernel, self).set_parent(ident, parent)
self.shell.set_parent(parent)
def send_multipart(self, msg, *args, **kwargs):
logger.debug('send_multipart %s %s %s %s', len(msg), msg, args, kwargs)
if len(msg) == 7:
msg0, msg1, msg2, msg3, msg4, msg5, msg6 = msg
logger.debug("msg0 %s", msg0)
logger.debug("msg1 %s", msg1)
logger.debug("msg2 %s", msg2)
logger.debug("msg3 %s", pformat(json.loads(msg3)))
logger.debug("msg4 %s", pformat(json.loads(msg4)))
logger.debug("msg5 %s", pformat(json.loads(msg5)))
logger.debug("msg6 %s", pformat(json.loads(msg6)))
msg3_data = json.loads(msg3)
msg6_data = json.loads(msg6)
if msg0.startswith(b"comm"):
_, _, comm_id = msg0.partition('-')
if msg3_data['msg_type'] == 'comm_open' and msg6_data[
'comm_id'] == comm_id:
self.update_widget(
comm_id,
msg6_data.get('data', {}).get('state', {}))
logger.debug("new widget %s %s", comm_id,
pformat(self.widgets[comm_id]))
if msg3_data['msg_type'] == 'comm_msg' and msg6_data[
'comm_id'] == comm_id:
if msg6_data.get('data', {}).get('method') == 'update':
self.update_widget(
comm_id,
msg6_data.get('data', {}).get('state', {}))
logger.debug("update widget %s %s", comm_id,
pformat(self.widgets[comm_id]))
def update_widget(self, comm_id, state):
self.widgets[comm_id].update(state)
self.widgets[comm_id]['widget_update_order'] = self.widget_update_order
self.widget_update_order += 1
def comm_open(self, stream, ident, msg):
logger.debug("comm_open: %s %s", ident, msg)
self.comm_manager.comm_open(stream, ident, msg)
def comm_msg(self, stream, ident, msg):
logger.debug("comm_msg: %s %s", ident, msg)
logger.debug("msg %s", pformat(msg))
comm_id = msg.get('content', {}).get('comm_id', {})
if comm_id in self.widgets:
self.widgets[comm_id].update(
msg.get('content', {}).get('data', {}).get('state', {}))
logger.debug("updated widget %s %s", comm_id,
self.widgets[comm_id])
self.comm_manager.comm_msg(stream, ident, msg)
def comm_close(self, stream, ident, msg):
logger.debug("comm_close: %s %s", ident, msg)
self.comm_manager.comm_close(stream, ident, msg)
class MetaKernel(Kernel):
"""The base MetaKernel class."""
app_name = 'metakernel'
identifier_regex = r'[^\d\W][\w\.]*'
func_call_regex = r'([^\d\W][\w\.]*)\([^\)\()]*\Z'
magic_prefixes = dict(magic='%', shell='!', help='?')
help_suffix = '?'
help_links = [
{
'text': "MetaKernel Magics",
'url': "https://metakernel.readthedocs.io/en/latest/source/README.html",
},
]
language_info = {
# 'mimetype': 'text/x-python',
# 'name': 'python',
# ------ If different from 'language':
# 'codemirror_mode': {
# "version": 2,
# "name": "ipython"
# }
# 'pygments_lexer': 'language',
# 'version' : "x.y.z",
# 'file_extension': '.py',
'help_links': help_links,
}
plot_settings = Dict(dict(backend='inline')).tag(config=True)
meta_kernel = None
@classmethod
def run_as_main(cls, *args, **kwargs):
"""Launch or install a metakernel.
Modules implementing a metakernel subclass can use the following lines:
if __name__ == '__main__':
MetaKernelSubclass.run_as_main()
"""
kwargs['app_name'] = cls.app_name
MetaKernelApp.launch_instance(kernel_class=cls, *args, **kwargs)
def __init__(self, *args, **kwargs):
super(MetaKernel, self).__init__(*args, **kwargs)
if MetaKernel.meta_kernel is None:
MetaKernel.meta_kernel = self
if self.log is None:
# This occurs if we call as a stand-alone kernel
# (eg, not as a process)
# FIXME: take care of input/output, eg StringIO
# make work without a session
self.log = logging.Logger(".metakernel")
else:
# Write has already been set
try:
sys.stdout.write = self.Write
except:
pass # Can't change stdout
self.redirect_to_log = False
self.shell = None
self.sticky_magics = OrderedDict()
self._i = None
self._ii = None
self._iii = None
self._ = None
self.__ = None
self.___ = None
self.max_hist_cache = 1000
self.hist_cache = []
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
self.hist_file = get_history_file(self)
self.parser = Parser(self.identifier_regex, self.func_call_regex,
self.magic_prefixes, self.help_suffix)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type)
self._ipy_formatter = IPythonDisplayFormatter()
self.env = {}
self.reload_magics()
# provide a way to get the current instance
self.set_variable("kernel", self)
# Run command line filenames, if given:
if self.parent is not None and self.parent.extra_args:
level = self.log.level
self.log.setLevel("INFO")
self.redirect_to_log = True
self.Write("Executing files...")
for filename in self.parent.extra_args:
self.Write(" %s..." % filename)
try:
self.do_execute_file(filename)
except Exception as exc:
self.log.info(" %s" % (exc,))
self.Write("Executing files: done!")
self.log.setLevel(level)
self.redirect_to_log = False
def makeSubkernel(self, kernel):
"""
Run this method in an IPython kernel to set
this kernel's input/output settings.
"""
from IPython import get_ipython
from IPython.display import display
shell = get_ipython()
if shell: # we are running under an IPython kernel
self.session = shell.kernel.session
self.Display = display
self.send_response = self._send_shell_response
else:
self.session = kernel.session
self.send_response = kernel.send_response
self.Display = kernel.Display
#####################################
# Methods which provide kernel - specific behavior
def set_variable(self, name, value):
"""
Set a variable to a Python-typed value.
"""
pass
def get_variable(self, name):
"""
Lookup a variable name and return a Python-typed value.
"""
pass
def repr(self, item):
"""The repr of the kernel."""
return repr(item)
def get_usage(self):
"""Get the usage statement for the kernel."""
return "This is a usage statement."
def get_kernel_help_on(self, info, level=0, none_on_fail=False):
"""Get help on an object. Called by the help magic."""
if none_on_fail:
return None
else:
return "Sorry, no help is available on '%s'." % info['code']
def handle_plot_settings(self):
"""Handle the current plot settings"""
pass
def get_local_magics_dir(self):
"""
Returns the path to local magics dir (eg ~/.ipython/metakernel/magics)
"""
base = get_ipython_dir()
return os.path.join(base, 'metakernel', 'magics')
def get_completions(self, info):
"""
Get completions from kernel based on info dict.
"""
return []
def do_execute_direct(self, code, silent=False):
"""
Execute code in the kernel language.
"""
pass
def do_execute_file(self, filename):
"""
Default code for running a file. Just opens the file, and sends
the text to do_execute_direct.
"""
code = "".join(open(filename).readlines())
return self.do_execute_direct(code)
def do_execute_meta(self, code):
"""
Execute meta code in the kernel. This uses the execute infrastructure
but allows JavaScript to talk directly to the kernel bypassing normal
processing.
When responding to the %%debug magic, the step and reset meta
commands can answer with a string in the format:
"highlight: [start_line, start_col, end_line, end_col]"
for highlighting expressions in the frontend.
"""
if code == "reset":
raise Exception("This kernel does not implement this meta command")
elif code == "stop":
raise Exception("This kernel does not implement this meta command")
elif code == "step":
raise Exception("This kernel does not implement this meta command")
elif code.startswith("inspect "):
raise Exception("This kernel does not implement this meta command")
else:
raise Exception("Unknown meta command: '%s'" % code)
def initialize_debug(self, code):
"""
This function is used with the %%debug magic for highlighting
lines of code, and for initializing debug functions.
Return the empty string if highlighting is not supported.
"""
#return "highlight: [%s, %s, %s, %s]" % (line1, col1, line2, col2)
return ""
def do_function_direct(self, function_name, arg):
"""
Call a function in the kernel language with args (as a single item).
"""
f = self.do_execute_direct(function_name)
return f(arg)
def restart_kernel(self):
"""Restart the kernel"""
pass
############################################
# Implement base class methods
def do_execute(self, code, silent=False, store_history=True, user_expressions=None,
allow_stdin=False):
"""Handle code execution.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#execute
"""
# Set the ability for the kernel to get standard-in:
self._allow_stdin = allow_stdin
# Create a default response:
self.kernel_resp = {
'status': 'ok',
# The base class increments the execution count
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
}
# TODO: remove this when IPython fixes this
# This happens at startup when the language is set to python
if '_usage.page_guiref' in code:
return self.kernel_resp
if code and store_history:
self.hist_cache.append(code.strip())
if not code.strip():
return self.kernel_resp
info = self.parse_code(code)
self.payload = []
retval = None
if info['magic'] and info['magic']['name'] == 'help':
if info['magic']['type'] == 'line':
level = 0
else:
level = 1
text = self.get_help_on(code, level)
if text:
content = {
"start_line_number": 0,
"source": "page",
}
if isinstance(text, dict):
content["data"] = text ## {mime-type: ..., mime-type:...}
self.log.debug(str(text))
else:
content["data"] = {"text/plain": text}
self.log.debug(text)
self.payload = [content]
elif info['magic'] or self.sticky_magics:
retval = None
if self.sticky_magics:
magics, code = _split_magics_code(code, self.magic_prefixes)
code = magics + self._get_sticky_magics() + code
stack = []
# Handle magics:
magic = None
prefixes = ((self.magic_prefixes['shell'],
self.magic_prefixes['magic']))
while code.startswith(prefixes):
magic = self.get_magic(code)
if magic is not None:
stack.append(magic)
code = magic.get_code()
# signal to exit, maybe error or no block
if not magic.evaluate:
break
else:
break
# Execute code, if any:
if ((magic is None or magic.evaluate) and code.strip() != ""):
if code.startswith("~~META~~:"):
retval = self.do_execute_meta(code[9:].strip())
else:
retval = self.do_execute_direct(code)
# Post-process magics:
for magic in reversed(stack):
retval = magic.post_process(retval)
else:
if code.startswith("~~META~~:"):
retval = self.do_execute_meta(code[9:].strip())
else:
retval = self.do_execute_direct(code)
self.post_execute(retval, code, silent)
if 'payload' in self.kernel_resp:
self.kernel_resp['payload'] = self.payload
return self.kernel_resp
def post_execute(self, retval, code, silent):
"""Post-execution actions
Handle special kernel variables and display response if not silent.
"""
# Handle in's
self.set_variable("_iii", self._iii)
self.set_variable("_ii", self._ii)
self.set_variable("_i", code)
self.set_variable("_i" + str(self.execution_count), code)
self._iii = self._ii
self._ii = code
if (retval is not None):
# --------------------------------------
# Handle out's (only when non-null)
self.set_variable("___", self.___)
self.set_variable("__", self.__)
self.set_variable("_", retval)
self.set_variable("_" + str(self.execution_count), retval)
self.___ = self.__
self.__ = retval
self.log.debug(retval)
if isinstance(retval, ExceptionWrapper):
self.kernel_resp['status'] = 'error'
content = {
'traceback': retval.traceback,
'evalue': retval.evalue,
'ename': retval.ename,
}
self.kernel_resp.update(content)
if not silent:
self.send_response(self.iopub_socket, 'error', content)
else:
try:
data = _formatter(retval, self.repr)
except Exception as e:
self.Error(e)
return
content = {
'execution_count': self.execution_count,
'data': data[0],
'metadata': data[1],
}
if not silent:
if Widget and isinstance(retval, Widget):
self.Display(retval)
return
self.send_response(self.iopub_socket, 'execute_result', content)
def do_history(self, hist_access_type, output, raw, session=None,
start=None, stop=None, n=None, pattern=None, unique=False):
"""
Access history at startup.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#history
"""
with open(self.hist_file) as fid:
self.hist_cache = json.loads(fid.read() or "[]")
return {'history': [(None, None, h) for h in self.hist_cache]}
def do_shutdown(self, restart):
"""
Shut down the app gracefully, saving history.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#kernel-shutdown
"""
if self.hist_file:
with open(self.hist_file, "w") as fid:
json.dump(self.hist_cache[-self.max_hist_cache:], fid)
if restart:
self.Print("Restarting kernel...")
self.restart_kernel()
self.reload_magics()
self.Print("Done!")
return {'status': 'ok', 'restart': restart}
def do_is_complete(self, code):
"""
Given code as string, returns dictionary with 'status' representing
whether code is ready to evaluate. Possible values for status are:
'complete' - ready to evaluate
'incomplete' - not yet ready
'invalid' - invalid code
'unknown' - unknown; the default unless overridden
Optionally, if 'status' is 'incomplete', you may indicate
an indentation string.
Example:
return {'status' : 'incomplete',
'indent': ' ' * 4}
https://jupyter-client.readthedocs.io/en/stable/messaging.html#code-completeness
"""
if code.startswith(self.magic_prefixes['magic']):
## force requirement to end with an empty line
if code.endswith("\n"):
return {'status' : 'complete'}
else:
return {'status' : 'incomplete'}
# otherwise, how to know is complete?
elif code.endswith("\n"):
return {'status' : 'complete'}
else:
return {'status' : 'incomplete'}
def do_complete(self, code, cursor_pos):
"""Handle code completion for the kernel.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#completion
"""
info = self.parse_code(code, 0, cursor_pos)
content = {
'matches': [],
'cursor_start': info['start'],
'cursor_end': info['end'],
'status': 'ok'
}
matches = info['path_matches']
if info['magic']:
# if the last line contains another magic, use that
line_info = self.parse_code(info['line'])
if line_info['magic']:
info = line_info
if info['magic']['type'] == 'line':
magics = self.line_magics
else:
magics = self.cell_magics
if info['magic']['name'] in magics:
magic = magics[info['magic']['name']]
info = info['magic']
if info['type'] == 'cell' and info['code']:
info = self.parse_code(info['code'])
else:
info = self.parse_code(info['args'])
matches.extend(magic.get_completions(info))
elif not info['magic']['code'] and not info['magic']['args']:
matches = []
for name in magics.keys():
if name.startswith(info['magic']['name']):
pre = info['magic']['prefix']
matches.append(pre + name)
info['start'] -= len(pre)
info['full_obj'] = pre + info['full_obj']
info['obj'] = pre + info['obj']
else:
matches.extend(self.get_completions(info))
if info['full_obj'] and len(info['full_obj']) > len(info['obj']):
new_list = [m for m in matches if m.startswith(info['full_obj'])]
if new_list:
content['cursor_end'] = (content['cursor_end'] +
len(info['full_obj']) -
len(info['obj']))
matches = new_list
content["matches"] = sorted(matches)
return content
def do_inspect(self, code, cursor_pos, detail_level=0):
"""Object introspection.
https://jupyter-client.readthedocs.io/en/stable/messaging.html#introspection
"""
if cursor_pos > len(code):
return
content = {'status': 'aborted', 'data': {}, 'found': False}
docstring = self.get_help_on(code, detail_level, none_on_fail=True,
cursor_pos=cursor_pos)
if docstring:
content["status"] = "ok"
content["found"] = True
if isinstance(docstring, dict): ## {"text/plain": ..., mime-type: ...}
content["data"] = docstring
self.log.debug(str(docstring))
else:
content["data"] = {"text/plain": docstring}
self.log.debug(docstring)
return content
def clear_output(self, wait=False):
"""Clear the output of the kernel."""
self.send_response(self.iopub_socket, 'clear_output',
{'wait': wait})
def Display(self, *objects, **kwargs):
"""Display one or more objects using rich display.
Supports a `clear_output` keyword argument that clears the output before displaying.
See https://ipython.readthedocs.io/en/stable/config/integrating.html?highlight=display#rich-display
"""
if kwargs.get('clear_output'):
self.clear_output(wait=True)
for item in objects:
if Widget and isinstance(item, Widget):
self.log.debug('Display Widget')
self._ipy_formatter(item)
else:
self.log.debug('Display Data')
try:
data = _formatter(item, self.repr)
except Exception as e:
self.Error(e)
return
content = {
'data': data[0],
'metadata': data[1]
}
self.send_response(
self.iopub_socket,
'display_data',
content
)
def Print(self, *objects, **kwargs):
"""Print `objects` to the iopub stream, separated by `sep` and followed by `end`.
Items can be strings or `Widget` instances.
"""
for item in objects:
if Widget and isinstance(item, Widget):
self.Display(item)
objects = [i for i in objects if not isinstance(i, Widget)]
message = format_message(*objects, **kwargs)
stream_content = {
'name': 'stdout', 'text': message}
self.log.debug('Print: %s' % message.rstrip())
if self.redirect_to_log:
self.log.info(message.rstrip())
else:
self.send_response(self.iopub_socket, 'stream', stream_content)
def Write(self, message):
"""Write message directly to the iopub stdout with no added end character."""
stream_content = {
'name': 'stdout', 'text': message}
self.log.debug('Write: %s' % message)
if self.redirect_to_log:
self.log.info(message)
else:
self.send_response(self.iopub_socket, 'stream', stream_content)
def Error(self, *objects, **kwargs):
"""Print `objects` to stdout, separated by `sep` and followed by `end`.
Objects are cast to strings.
"""
message = format_message(*objects, **kwargs)
self.log.debug('Error: %s' % message.rstrip())
stream_content = {
'name': 'stderr',
'text': RED + message + NORMAL
}
if self.redirect_to_log:
self.log.info(message.rstrip())
else:
self.send_response(self.iopub_socket, 'stream', stream_content)
##############################
# Private API and methods not likely to be overridden
def reload_magics(self):
"""Reload all of the line and cell magics."""
self.line_magics = {}
self.cell_magics = {}
# get base magic files and those relative to the current class
# directory
magic_files = []
# Make a metakernel/magics if it doesn't exist:
local_magics_dir = get_local_magics_dir()
# Search all of the places there could be magics:
try:
paths = [os.path.join(os.path.dirname(
os.path.abspath(inspect.getfile(self.__class__))), "magics")]
except:
paths = []
paths += [local_magics_dir,
os.path.join(os.path.dirname(os.path.abspath(__file__)), "magics")]
for magic_dir in paths:
sys.path.append(magic_dir)
magic_files.extend(glob.glob(os.path.join(magic_dir, "*.py")))
for magic in magic_files:
basename = os.path.basename(magic)
if basename == "__init__.py":
continue
try:
module = __import__(os.path.splitext(basename)[0])
imp.reload(module)
module.register_magics(self)
except Exception as e:
self.log.error("Can't load '%s': error: %s" % (magic, e))
def register_magics(self, magic_klass):
"""Register magics for a given magic_klass."""
magic = magic_klass(self)
line_magics = magic.get_magics('line')
cell_magics = magic.get_magics('cell')
for name in line_magics:
self.line_magics[name] = magic
for name in cell_magics:
self.cell_magics[name] = magic
def send_response(self, *args, **kwargs):
### if we are running via %parallel, we might not have a
### session
if self.session:
super(MetaKernel, self).send_response(*args, **kwargs)
def call_magic(self, line):
"""
Given an line, such as "%download http://example.com/", parse
and execute magic.
"""
return self.get_magic(line)
def get_magic(self, text):
## FIXME: Bad name, use call_magic instead.
# if first line matches a magic,
# call magic.call_magic() and return magic object
info = self.parse_code(text)
magic = self.line_magics['magic']
return magic.get_magic(info)
def get_magic_args(self, text):
# if first line matches a magic,
# call magic.call_magic() and return magic args
info = self.parse_code(text)
magic = self.line_magics['magic']
return magic.get_magic(info, get_args=True)
def get_help_on(self, expr, level=0, none_on_fail=False,
cursor_pos=-1):
"""Get help for an expression using the help magic."""
help_magic = self.line_magics['help']
return help_magic.get_help_on(expr, level, none_on_fail, cursor_pos)
def parse_code(self, code, cursor_start=0, cursor_end=-1):
"""Parse code using our parser."""
return self.parser.parse_code(code, cursor_start, cursor_end)
def _get_sticky_magics(self):
retval = ""
for key in self.sticky_magics:
retval += (key + " " + self.sticky_magics[key] + "\n")
return retval
def _send_shell_response(self, socket, stream_type, content):
publish_display_data({ 'text/plain': content['text'] })
class MetaKernel(Kernel):
identifier_regex = r'[^\d\W][\w\.]*'
func_call_regex = r'([^\d\W][\w\.]*)\([^\)\()]*\Z'
magic_prefixes = dict(magic='%', shell='!', help='?')
help_suffix = '?'
help_links = [
{
'text':
"MetaKernel Magics",
'url':
"https://github.com/calysto/metakernel/blob/master/metakernel/magics/README.md",
},
]
language_info = {
# 'mimetype': 'text/x-python',
# 'name': 'python',
# ------ If different from 'language':
# 'codemirror_mode': {
# "version": 2,
# "name": "ipython"
# }
# 'pygments_lexer': 'language',
# 'version' : "x.y.z",
# 'file_extension': '.py',
'help_links': help_links,
}
meta_kernel = None
def __init__(self, *args, **kwargs):
super(MetaKernel, self).__init__(*args, **kwargs)
if MetaKernel.meta_kernel is None:
MetaKernel.meta_kernel = self
if self.log is None:
# This occurs if we call as a stand-alone kernel
# (eg, not as a process)
# FIXME: take care of input/output, eg StringIO
# make work without a session
self.log = logging.Logger(".metakernel")
else:
# Write has already been set
try:
sys.stdout.write = self.Write
except:
pass # Can't change stdout
self.sticky_magics = {}
self._i = None
self._ii = None
self._iii = None
self._ = None
self.__ = None
self.___ = None
self.max_hist_cache = 1000
self.hist_cache = []
self.comm_manager = CommManager(shell=None, parent=self, kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
self.plot_settings = dict(backend='inline')
self.hist_file = get_history_file(self)
self.reload_magics()
# provide a way to get the current instance
self.set_variable("kernel", self)
self.parser = Parser(self.identifier_regex, self.func_call_regex,
self.magic_prefixes, self.help_suffix)
self.comm_manager = CommManager(shell=None, parent=self, kernel=self)
self.comm_manager.register_target('ipython.widget',
lazy_import_handle_comm_opened)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager,
msg_type)
self._ipy_formatter = IPythonDisplayFormatter()
self.env = {}
@classmethod
def subkernel(cls, kernel):
"""
Handle issues regarding making a kernel a subkernel to this
one. Used in %parallel and %kernel.
"""
pass
def makeSubkernelTo(self, main_kernel, display_function):
"""
Handle issues regarding making this kernel be a subkernel to
another. Used in making metakernels be magics for IPython.
"""
self.session = main_kernel.session
self.Display = display_function
def makeSubkernelToIPython(self):
"""
Run this method in an IPython kernel to set
this kernel's input/output settings.
"""
from IPython import get_ipython
from IPython.display import display
ip = get_ipython()
if ip: # we are running under an IPython kernel
self.makeSubkernelTo(ip.parent, display)
else:
raise Exception("Need to run under an IPython kernel")
#####################################
# Methods which provide kernel - specific behavior
def set_variable(self, name, value):
"""
Set a variable to a Python-typed value.
"""
pass
def get_variable(self, name):
"""
Lookup a variable name and return a Python-typed value.
"""
pass
def repr(self, item):
return repr(item)
def get_usage(self):
return "This is a usage statement."
def get_kernel_help_on(self, info, level=0, none_on_fail=False):
if none_on_fail:
return None
else:
return "Sorry, no help is available on '%s'." % info['code']
def handle_plot_settings(self):
"""Handle the current plot settings"""
pass
def get_local_magics_dir(self):
"""
Returns the path to local magics dir (eg ~/.ipython/metakernel/magics)
"""
base = get_ipython_dir()
return os.path.join(base, 'metakernel', 'magics')
def get_completions(self, info):
"""
Get completions from kernel based on info dict.
"""
return []
def do_execute_direct(self, code, silent=False):
"""
Execute code in the kernel language.
"""
pass
def do_execute_file(self, filename):
"""
Default code for running a file. Just opens the file, and sends
the text to do_execute_direct.
"""
code = "".join(open(filename).readlines())
return self.do_execute_direct(code)
def do_execute_meta(self, code):
"""
Execute meta code in the kernel. This uses the execute infrastructure
but allows JavaScript to talk directly to the kernel bypassing normal
processing.
When responding to the %%debug magic, the step and reset meta
commands can answer with a string in the format:
"highlight: [start_line, start_col, end_line, end_col]"
for highlighting expressions in the frontend.
"""
if code == "reset":
raise Exception("This kernel does not implement this meta command")
elif code == "stop":
raise Exception("This kernel does not implement this meta command")
elif code == "step":
raise Exception("This kernel does not implement this meta command")
elif code.startswith("inspect "):
raise Exception("This kernel does not implement this meta command")
else:
raise Exception("Unknown meta command: '%s'" % code)
def initialize_debug(self, code):
"""
This function is used with the %%debug magic for highlighting
lines of code, and for initializing debug functions.
Return the empty string if highlighting is not supported.
"""
#return "highlight: [%s, %s, %s, %s]" % (line1, col1, line2, col2)
return ""
def do_function_direct(self, function_name, arg):
"""
Call a function in the kernel language with args (as a single item).
"""
self.Error("This language does not support \"%pmap function args\".")
def restart_kernel(self):
"""Restart the kernel"""
pass
############################################
# Implement base class methods
def do_execute(self,
code,
silent=False,
store_history=True,
user_expressions=None,
allow_stdin=False):
# Set the ability for the kernel to get standard-in:
self._allow_stdin = allow_stdin
# Create a default response:
self.kernel_resp = {
'status': 'ok',
# The base class increments the execution count
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
}
# TODO: remove this when IPython fixes this
# This happens at startup when the language is set to python
if '_usage.page_guiref' in code:
return self.kernel_resp
if code and store_history:
self.hist_cache.append(code.strip())
if not code.strip():
return self.kernel_resp
info = self.parse_code(code)
self.payload = []
retval = None
if info['magic'] and info['magic']['name'] == 'help':
if info['magic']['type'] == 'line':
level = 0
else:
level = 1
text = self.get_help_on(code, level)
self.log.debug(text)
if text:
self.payload = [{
"data": {
"text/plain": text
},
"start_line_number": 0,
"source": "page"
}]
elif info['magic'] or self.sticky_magics:
retval = None
if self.sticky_magics:
magics, code = _split_magics_code(code, self.magic_prefixes)
code = magics + self._get_sticky_magics() + code
stack = []
# Handle magics:
magic = None
prefixes = ((self.magic_prefixes['shell'],
self.magic_prefixes['magic']))
while code.startswith(prefixes):
magic = self.get_magic(code)
if magic is not None:
stack.append(magic)
code = magic.get_code()
# signal to exit, maybe error or no block
if not magic.evaluate:
break
else:
break
# Execute code, if any:
if ((magic is None or magic.evaluate) and code.strip() != ""):
if code.startswith("~~META~~:"):
retval = self.do_execute_meta(code[9:].strip())
else:
retval = self.do_execute_direct(code)
# Post-process magics:
for magic in reversed(stack):
retval = magic.post_process(retval)
else:
if code.startswith("~~META~~:"):
retval = self.do_execute_meta(code[9:].strip())
else:
retval = self.do_execute_direct(code)
self.post_execute(retval, code, silent)
if 'payload' in self.kernel_resp:
self.kernel_resp['payload'] = self.payload
return self.kernel_resp
def post_execute(self, retval, code, silent):
# Handle in's
self.set_variable("_iii", self._iii)
self.set_variable("_ii", self._ii)
self.set_variable("_i", code)
self.set_variable("_i" + str(self.execution_count), code)
self._iii = self._ii
self._ii = code
if (retval is not None):
# --------------------------------------
# Handle out's (only when non-null)
self.set_variable("___", self.___)
self.set_variable("__", self.__)
self.set_variable("_", retval)
self.set_variable("_" + str(self.execution_count), retval)
self.___ = self.__
self.__ = retval
self.log.debug(retval)
try:
content = {
'execution_count': self.execution_count,
'data': _formatter(retval, self.repr),
'metadata': dict()
}
except Exception as e:
self.Error(e)
return
if not silent:
if Widget and isinstance(retval, Widget):
self.Display(retval)
return
self.send_response(self.iopub_socket, 'execute_result',
content)
def do_history(self,
hist_access_type,
output,
raw,
session=None,
start=None,
stop=None,
n=None,
pattern=None,
unique=False):
"""
Access history at startup.
"""
if not self.hist_file:
return {'history': []}
# else:
if not os.path.exists(self.hist_file):
with open(self.hist_file, 'wb') as fid:
fid.write('')
with open(self.hist_file, 'rb') as fid:
history = fid.read().decode('utf-8', 'replace')
history = history.splitlines()
history = history[:self.max_hist_cache]
self.hist_cache = history
history = [(None, None, h) for h in history]
return {'history': history}
def do_shutdown(self, restart):
"""
Shut down the app gracefully, saving history.
"""
if self.hist_file:
with open(self.hist_file, 'wb') as fid:
data = '\n'.join(self.hist_cache[-self.max_hist_cache:])
fid.write(data.encode('utf-8'))
if restart:
self.Print("Restarting kernel...")
self.reload_magics()
self.restart_kernel()
self.Print("Done!")
return {'status': 'ok', 'restart': restart}
def do_is_complete(self, code):
"""
Given code as string, returns dictionary with 'status' representing
whether code is ready to evaluate. Possible values for status are:
'complete' - ready to evaluate
'incomplete' - not yet ready
'invalid' - invalid code
'unknown' - unknown; the default unless overridden
Optionally, if 'status' is 'incomplete', you may indicate
an indentation string.
Example:
return {'status' : 'incomplete',
'indent': ' ' * 4}
"""
return {'status': 'unknown'}
def do_complete(self, code, cursor_pos):
info = self.parse_code(code, 0, cursor_pos)
content = {
'matches': [],
'cursor_start': info['start'],
'cursor_end': info['end'],
'metadata': {},
'status': 'ok'
}
matches = info['path_matches']
if info['magic']:
# if the last line contains another magic, use that
line_info = self.parse_code(info['line'])
if line_info['magic']:
info = line_info
if info['magic']['type'] == 'line':
magics = self.line_magics
else:
magics = self.cell_magics
if info['magic']['name'] in magics:
magic = magics[info['magic']['name']]
info = info['magic']
if info['type'] == 'cell' and info['code']:
info = self.parse_code(info['code'])
else:
info = self.parse_code(info['args'])
matches.extend(magic.get_completions(info))
elif not info['magic']['code'] and not info['magic']['args']:
matches = []
for name in magics.keys():
if name.startswith(info['magic']['name']):
pre = info['magic']['prefix']
matches.append(pre + name)
info['start'] -= len(pre)
info['full_obj'] = pre + info['full_obj']
info['obj'] = pre + info['obj']
else:
matches.extend(self.get_completions(info))
if info['full_obj'] and len(info['full_obj']) > len(info['obj']):
new_list = [m for m in matches if m.startswith(info['full_obj'])]
if new_list:
content['cursor_end'] = (content['cursor_end'] +
len(info['full_obj']) -
len(info['obj']))
matches = new_list
content["matches"] = sorted(matches)
return content
def do_inspect(self, code, cursor_pos, detail_level=0):
# Object introspection
if cursor_pos > len(code):
return
content = {'status': 'aborted', 'data': {}, 'found': False}
docstring = self.get_help_on(code,
detail_level,
none_on_fail=True,
cursor_pos=cursor_pos)
if docstring:
content["data"] = {"text/plain": docstring}
content["status"] = "ok"
content["found"] = True
self.log.debug(docstring)
return content
##############################
# Private API and methods not likely to be overridden
def reload_magics(self):
self.line_magics = {}
self.cell_magics = {}
# get base magic files and those relative to the current class
# directory
magic_files = []
# Make a metakernel/magics if it doesn't exist:
local_magics_dir = get_local_magics_dir()
# Search all of the places there could be magics:
paths = [
local_magics_dir,
os.path.join(os.path.dirname(os.path.abspath(__file__)), "magics")
]
try:
paths += [
os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(self.__class__))),
"magics")
]
except:
pass
for magic_dir in paths:
sys.path.append(magic_dir)
magic_files.extend(glob.glob(os.path.join(magic_dir, "*.py")))
for magic in magic_files:
basename = os.path.basename(magic)
if basename == "__init__.py":
continue
try:
module = __import__(os.path.splitext(basename)[0])
imp.reload(module)
module.register_magics(self)
except Exception as e:
self.log.error("Can't load '%s': error: %s" % (magic, e))
def register_magics(self, magic_klass):
magic = magic_klass(self)
line_magics = magic.get_magics('line')
cell_magics = magic.get_magics('cell')
for name in line_magics:
self.line_magics[name] = magic
for name in cell_magics:
self.cell_magics[name] = magic
def clear_output(self, wait=False):
self.send_response(self.iopub_socket, 'clear_output', {'wait': wait})
def Display(self, *args, **kwargs):
clear_output = kwargs.get("clear_output", False)
for message in args:
if isinstance(message, HTML):
if clear_output:
self.send_response(self.iopub_socket, 'clear_output',
{'wait': True})
if Widget and isinstance(message, Widget):
self.log.debug('Display Widget')
self._ipy_formatter(message)
else:
self.log.debug('Display Data')
try:
data = _formatter(message, self.repr)
except Exception as e:
self.Error(e)
return
self.send_response(self.iopub_socket, 'display_data', {
'data': data,
'metadata': dict()
})
def Print(self, *args, **kwargs):
end = kwargs["end"] if ("end" in kwargs) else "\n"
message = ""
for item in args:
if Widget and isinstance(item, Widget):
self.Display(item)
else:
if message:
message += " "
if PY3:
message += str(item)
else:
message += codecs.encode(item, "utf-8")
message += end
stream_content = {
'name': 'stdout',
'text': message,
'metadata': dict()
}
self.log.debug('Print: %s' % message)
self.send_response(self.iopub_socket, 'stream', stream_content)
def Write(self, message):
stream_content = {
'name': 'stdout',
'text': message,
'metadata': dict()
}
self.log.debug('Write: %s' % message)
self.send_response(self.iopub_socket, 'stream', stream_content)
def Error(self, *args, **kwargs):
message = format_message(*args, **kwargs)
self.log.debug('Error: %s' % message)
stream_content = {
'name': 'stderr',
'text': message,
'metadata': dict()
}
self.send_response(self.iopub_socket, 'stream', stream_content)
def call_magic(self, line):
"""
Given an line, such as "%download http://example.com/", parse
and execute magic.
"""
return self.get_magic(line)
def get_magic(self, text):
## FIXME: Bad name, use call_magic instead.
# if first line matches a magic,
# call magic.call_magic() and return magic object
info = self.parse_code(text)
magic = self.line_magics['magic']
return magic.get_magic(info)
def get_magic_args(self, text):
# if first line matches a magic,
# call magic.call_magic() and return magic args
info = self.parse_code(text)
magic = self.line_magics['magic']
return magic.get_magic(info, get_args=True)
def get_help_on(self, expr, level=0, none_on_fail=False, cursor_pos=-1):
help_magic = self.line_magics['help']
return help_magic.get_help_on(expr, level, none_on_fail, cursor_pos)
def parse_code(self, code, cursor_start=0, cursor_end=-1):
return self.parser.parse_code(code, cursor_start, cursor_end)
def _get_sticky_magics(self):
retval = ""
for key in self.sticky_magics:
retval += (key + " " +
" ".join(self.sticky_magics[key])).strip() + "\n"
return retval
class polymakeKernel(Kernel):
implementation = 'jupyter_polymake_wrapper'
implementation_version = __version__
help_links = [ { 'text': "Polymake website", 'url': "http://polymake.org/" },
{ 'text': "Polymake documentation", 'url': "https://polymake.org/doku.php/documentation" },
{ 'text': "Polymake tutorial", 'url': "https://polymake.org/doku.php/tutorial/start" },
{ 'text': "Polymake reference", 'url': "https://polymake.org/release_docs/3.0/" } ]
def _replace_get_ipython(self):
new_kernel = own_ipython(self)
global kernel_object_for_ipython
kernel_object_for_ipython = new_kernel
@property
def language_version(self):
m = version_pat.search(self.banner)
return m.group(1)
_banner = None
@property
def banner(self):
if self._banner is None:
self._banner = "Jupyter kernel for polymake"
return self._banner
language_info = {'name': 'polymake',
'codemirror_mode': 'perl', #
'mimetype': 'text/x-polymake',
'file_extension': '.pl'} # FIXME: Is this even real?
def __init__(self, **kwargs):
Kernel.__init__(self, **kwargs)
self._replace_get_ipython()
self.comm_manager = CommManager(shell=None, parent=self,
kernel=self)
self.shell_handlers['comm_open'] = self.comm_manager.comm_open
self.shell_handlers['comm_msg'] = self.comm_manager.comm_msg
self.shell_handlers['comm_close'] = self.comm_manager.comm_close
if ipywidgets_extension_loaded:
self.comm_manager.register_target('ipython.widget', Widget.handle_comm_opened)
self._start_polymake()
def _start_polymake(self):
JuPyMake.InitializePolymake()
try:
self._run_polymake_command( 'include "common::jupyter.rules";' )
except PolymakeRunException:
return
return
def _run_polymake_command( self, code ):
try:
output = JuPyMake.ExecuteCommand( code.strip()+"\n" )
except Exception as exception:
raise PolymakeRunException(exception.args[0])
return output
def _process_python( self, code ):
if code.find( "@python" ) == -1 and code.find( "@widget" ) == -1:
return False
exec(code[7:],globals(),locals())
return True
def do_execute(self, code, silent, store_history=True,
user_expressions=None, allow_stdin=False):
default_return = {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
if not code.strip():
return default_return
if self._process_python( code ):
return default_return
interrupted = False
code = code.rstrip()
try:
output = self._run_polymake_command( code )
except KeyboardInterrupt:
self._run_polymake_command( '' )
interrupted = True
except PolymakeRunException as exception:
output = exception.args[0]
stream_content = {'execution_count': self.execution_count, 'data': { 'text/plain': "Error: Incomplete Statement:\n" + code } }
self.send_response( self.iopub_socket, 'execute_result', stream_content )
return {'status': 'error', 'execution_count': self.execution_count,
'ename': 'PolymakeRunException', 'evalue': output, 'traceback': []}
if not silent:
if output[0] == True:
if output[1] != "":
output_stdout = output[1]
while output_stdout.find( '.@@HTML@@' ) != -1:
html_position = output_stdout.find( '.@@HTML@@' )
html_end_position = output_stdout.find( '.@@ENDHTML@@' )
if html_position > 0:
before_html = output_stdout[:html_position].rstrip()
else:
before_html = ''
output_html = output_stdout[html_position+9:html_end_position-1].strip().rstrip()
output_stdout = output_stdout[html_end_position+12:].strip()
if before_html != '':
stream_content = {'execution_count': self.execution_count, 'data': { 'text/plain': before_html } }
self.send_response( self.iopub_socket, 'execute_result', stream_content )
stream_content = {'execution_count': self.execution_count,
'source' : "polymake",
#'data': { 'text/html': "Sorry, threejs visualization is currently not available"},
'data': { 'text/html': output_html},
'metadata': dict() }
self.send_response( self.iopub_socket, 'display_data', stream_content )
if len(output_stdout) != 0:
stream_content = {'execution_count': self.execution_count, 'data': { 'text/plain': output_stdout } }
self.send_response( self.iopub_socket, 'execute_result', stream_content )
if output[2] != "":
output_html = "<details><summary><pre style=\"display:inline\"><small>Click here for additional output</small></pre></summary>\n<pre>\n"+output[2]+"</pre>\n</details>\n"
stream_content = {'execution_count': self.execution_count,
'source' : "polymake",
'data': { 'text/html': output_html},
'metadata': dict() }
self.send_response( self.iopub_socket, 'display_data', stream_content )
if output[3] != "":
stream_content = {'execution_count': self.execution_count, 'data': { 'text/plain': output[3] } }
self.send_response( self.iopub_socket, 'execute_result', stream_content )
return {'status': 'error', 'execution_count': self.execution_count,
'ename': 'PolymakeRunException', 'evalue': output, 'traceback': []}
elif output[0] == False:
if output[3] == "":
stream_content = {'execution_count': self.execution_count, 'data': { 'text/plain': "Error: Incomplete Statement:\n" + code } }
self.send_response( self.iopub_socket, 'execute_result', stream_content )
return {'status': 'error', 'execution_count': self.execution_count,
'ename': 'IncompleteStatementError', 'evalue': output, 'traceback': []}
else:
stream_content = {'execution_count': self.execution_count, 'data': { 'text/plain': output[3] } }
self.send_response( self.iopub_socket, 'execute_result', stream_content )
return {'status': 'error', 'execution_count': self.execution_count,
'ename': 'PolymakeRunException', 'evalue': output, 'traceback': []}
if interrupted:
return {'status': 'abort', 'execution_count': self.execution_count}
return {'status': 'ok', 'execution_count': self.execution_count,
'payload': [], 'user_expressions': {}}
def do_shutdown(self, restart):
if restart:
self._start_polymake()
## Temporary method to determine offset of completion. Will be replaced soon.
def get_completion_length( self, code, completion ):
code_length = len(code)
maximal_length = 0
for i in range(1,code_length+1):
if code[code_length-i:code_length] == completion[0:i]:
maximal_length = i
return maximal_length
def do_complete(self, code, cursor_pos):
try:
completions = JuPyMake.GetCompletion(code[0:cursor_pos])
except:
completions = (0,"",[])
completion_offset = completions[0]
cur_start = cursor_pos - completion_offset
return {'matches': completions[2], 'cursor_start': cur_start,
'cursor_end': cursor_pos, 'metadata': dict(),
'status': 'ok'}
def do_is_complete( self, code ):
new_code = 'if(0){ ' + code + ' }'
try:
output = self._run_polymake_command( new_code )
except PolymakeRunException:
return {'status' : 'incomplete', 'indent': '' }
if output[0] == False:
return {'status' : 'incomplete', 'indent': '' }
return {'status' : 'complete' }
def do_inspect( self, code, cursor_pos, detail_level=0 ):
print(detail_level)
## ignore detail_level for now
full = True
try:
output = JuPyMake.GetContextHelp( input=code, position=cursor_pos, full=full )
except PolymakeRunException:
output = []
try:
output_html = JuPyMake.GetContextHelp( input=code, position=cursor_pos, full=full, html=True )
except PolymakeRunException:
output_html = []
output_data = { }
if output != []:
output_data['text/plain'] = "\n".join(output)
if output_html != []:
output_data['text/html'] = "\n".join(output_html)
return {'status': 'ok', 'data': output_data, 'metadata': {}, 'found': True}