def execute_notebook(nb_path):
"""Execute a notebook under grading conditions"""
graded_nb_path = os.path.splitext(nb_path)[0] + '-graded.ipynb'
nb_directory = os.path.split(nb_path)[0]
# read in input notebook and check the source for shenanigans
nb = nbformat.read(nb_path, as_version=4)
source = ""
for cell in nb.cells:
if cell.cell_type != "code":
continue
isp = IPythonInputSplitter(line_input_checker=False)
cell_source = isp.transform_cell(cell.source)
source += cell_source
tree = ast.parse(source)
# no points for you if you try and cheat
# XXX add a check for people importing a function called `check`
if find_check_assignment(tree) or find_check_definition(tree):
return
# run the notebook
with chdir(nb_directory):
pm.execute_notebook(nb_path, graded_nb_path)
graded_nb = nbformat.read(graded_nb_path, as_version=4)
return graded_nb
def execute_notebook(nb, initial_env=None, ignore_errors=False):
"""
Execute notebook & return the global environment that results from execution.
If ignore_errors is True, exceptions are swallowed.
nb is passed in as a dictionary that's a parsed ipynb file
"""
with hide_outputs():
if initial_env:
global_env = initial_env.copy()
else:
global_env = {}
for cell in nb['cells']:
if cell['cell_type'] == 'code':
# transform the input to executable Python
# FIXME: use appropriate IPython functions here
isp = IPythonInputSplitter(line_input_checker=False)
source = isp.transform_cell(''.join(cell['source']))
try:
with open(
'/dev/null',
'w') as f, redirect_stdout(f), redirect_stderr(f):
exec(source, global_env)
except:
if not ignore_errors:
raise
return global_env
class PyDevFrontEnd:
def __init__(self, *args, **kwargs):
# Initialization based on: from IPython.testing.globalipapp import start_ipython
self._curr_exec_line = 0
# Store certain global objects that IPython modifies
_displayhook = sys.displayhook
_excepthook = sys.excepthook
# Create and initialize our IPython instance.
shell = TerminalInteractiveShell.instance()
# Create an intput splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
shell.showtraceback = _showtraceback
# IPython is ready, now clean up some global state...
# Deactivate the various python system hooks added by ipython for
# interactive convenience so we don't confuse the doctest system
sys.displayhook = _displayhook
sys.excepthook = _excepthook
# So that ipython magics and aliases can be doctested (they work by making
# a call into a global _ip object). Also make the top-level get_ipython
# now return this without recursively calling here again.
get_ipython = shell.get_ipython
try:
import __builtin__
except:
import builtins as __builtin__
__builtin__._ip = shell
__builtin__.get_ipython = get_ipython
# We want to print to stdout/err as usual.
io.stdout = original_stdout
io.stderr = original_stderr
self.ipython = shell
def complete(self, string):
return self.ipython.complete(None, line=string)
def is_complete(self, string):
return not self.input_splitter.push_accepts_more()
def getNamespace(self):
return self.ipython.user_ns
def addExec(self, line):
self.input_splitter.push(line)
if self.is_complete(line):
self.ipython.run_cell(self.input_splitter.source_reset())
return False
else:
return True
class PyDevFrontEnd:
def __init__(self, *args, **kwargs):
# Initialization based on: from IPython.testing.globalipapp import start_ipython
self._curr_exec_line = 0
# Store certain global objects that IPython modifies
_displayhook = sys.displayhook
_excepthook = sys.excepthook
# Create and initialize our IPython instance.
shell = TerminalInteractiveShell.instance()
# Create an intput splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
shell.showtraceback = _showtraceback
# IPython is ready, now clean up some global state...
# Deactivate the various python system hooks added by ipython for
# interactive convenience so we don't confuse the doctest system
sys.displayhook = _displayhook
sys.excepthook = _excepthook
# So that ipython magics and aliases can be doctested (they work by making
# a call into a global _ip object). Also make the top-level get_ipython
# now return this without recursively calling here again.
get_ipython = shell.get_ipython
try:
import __builtin__
except:
import builtins as __builtin__
__builtin__._ip = shell
__builtin__.get_ipython = get_ipython
# We want to print to stdout/err as usual.
io.stdout = original_stdout
io.stderr = original_stderr
self.ipython = shell
def complete(self, string):
return self.ipython.complete(None, line=string)
def is_complete(self, string):
return not self.input_splitter.push_accepts_more()
def getNamespace(self):
return self.ipython.user_ns
def addExec(self, line):
self.input_splitter.push(line)
if self.is_complete(line):
self.ipython.run_cell(self.input_splitter.source_reset())
return False
else:
return True
def execute(code, namespace):
isp = IPythonInputSplitter()
kernel_manager = MyInProcessKernelManager()
kernel_manager.start_kernel(namespace)
kernel = kernel_manager.kernel
def do_execute():
result = kernel.shell.run_cell(cell, False)
if result.error_before_exec is not None:
err = result.error_before_exec
else:
err = result.error_in_exec
if not result.success:
if kernel.shell._last_traceback:
for tb in kernel.shell._last_traceback:
print(tb) #TODO: log
for line in code.splitlines():
if isp.push_accepts_more():
isp.push(line.strip("\n"))
continue
cell = isp.source_reset()
do_execute()
isp.push(line.strip("\n"))
cell = isp.source_reset()
if len(cell):
do_execute()
return namespace
def _ipython_to_python(source):
try:
from IPython.core.inputsplitter import IPythonInputSplitter
except ImportError:
log.warning("IPython is required to process Notebook source code - "
"install it by running 'pip install ipython'")
raise MissingIPython()
else:
isp = IPythonInputSplitter(line_input_checker=False)
python_source = isp.transform_cell(source)
return _fix_ipython_to_python_lf(python_source, source)
def ipython2encodedpython(code):
def tweak_transform(orig_transform):
"""
Takes the transform and modifies it such that we compare each line to its transformation. If they are
different, that means the line is a special ipython command. We strip that from the output, but record the
special command in a comment so we can restore it.
"""
def new_push_builder(push_func):
def new_push(line):
result = push_func(line)
if line != result:
return "# EPY: ESCAPE {}".format(line)
return result
return new_push
orig_transform.push = functools.update_wrapper(
new_push_builder(orig_transform.push), orig_transform.push)
return orig_transform
from IPython.core.inputtransformer import StatelessInputTransformer
@StatelessInputTransformer.wrap
def escaped_epy_lines(line):
"""Transform lines that happen to look like EPY comments."""
if line.startswith("# EPY"):
return "# EPY: ESCAPE {}".format(line)
return line
"""Transform IPython syntax to an encoded Python syntax
Parameters
----------
code : str
IPython code, to be transformed to Python encoded in a way to facilitate transformation back into IPython.
"""
from IPython.core.inputsplitter import IPythonInputSplitter
# get a list of default line transforms. then capture
fake_isp = IPythonInputSplitter(line_input_checker=False)
logical_line_transforms = [escaped_epy_lines()]
logical_line_transforms.extend([
tweak_transform(transform)
for transform in fake_isp.logical_line_transforms
])
isp = IPythonInputSplitter(line_input_checker=False,
logical_line_transforms=logical_line_transforms)
result = isp.transform_cell(code)
return result
def _is_pdb_complete(self, source):
"""
Check if the pdb input is ready to be executed.
"""
if source and source[0] == '!':
source = source[1:]
if PY2:
tm = IPythonInputSplitter()
else:
tm = TransformerManager()
complete, indent = tm.check_complete(source)
if indent is not None:
indent = indent * ' '
return complete != 'incomplete', indent
def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False):
"""
Execute notebook & return the global environment that results from execution.
TODO: write a note about the injection of check_results
If ignore_errors is True, exceptions are swallowed.
secret contains random digits so check_results and check are not easily modifiable
nb is passed in as a dictionary that's a parsed ipynb file
"""
with hide_outputs():
if initial_env:
global_env = initial_env.copy()
else:
global_env = {}
source = ""
# Before rewriting AST, find cells of code that generate errors.
# One round of execution is done beforehand to mimic the Jupyter notebook style of running
# (e.g. code runs up to the point of execution).
# The reason this is workaround is introduced is because once the
# source code is parsed into an AST, there is no sense of local cells
exec("from gofer.ok import check", global_env)
exec("from IPython.display import display", global_env)
for cell in nb['cells']:
if cell['cell_type'] == 'code':
# transform the input to executable Python
# FIXME: use appropriate IPython functions here
isp = IPythonInputSplitter(line_input_checker=False)
try:
code_lines = []
for line in cell['source']:
# Filter out ipython magic commands
if not line.startswith('%'):
code_lines.append(ok_grade_to_check(line))
cell_source = isp.transform_cell(''.join(code_lines))
# exec(cell_source, global_env)
run_this_block(cell_source, secret, global_env, ignore_errors)
source += cell_source
except Exception as e:
logging.error(e)
traceback.print_exc()
if not ignore_errors:
raise
return global_env
class IPythonValidator(PythonValidator):
def __init__(self, *args, **kwargs):
super(IPythonValidator, self).__init__(*args, **kwargs)
self.isp = IPythonInputSplitter()
def validate(self, document):
document = Document(text=self.isp.transform_cell(document.text))
super(IPythonValidator, self).validate(document)
def __init__(self, pydev_host, pydev_client_port, *args, **kwarg):
# Create and initialize our IPython instance.
self.ipython = PyDevTerminalInteractiveShell.instance()
# Back channel to PyDev to open editors (in the future other
# info may go back this way. This is the same channel that is
# used to get stdin, see StdIn in pydev_console_utils)
self.ipython.set_hook(
'editor', create_editor_hook(pydev_host, pydev_client_port))
# Create an input splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
# Display the IPython banner, this has version info and
# help info
self.ipython.show_banner()
class IPythonValidator(PythonValidator):
def __init__(self, *args, **kwargs):
super(IPythonValidator, self).__init__(*args, **kwargs)
self.isp = IPythonInputSplitter()
def validate(self, document):
document = Document(text=self.isp.transform_cell(document.text))
super(IPythonValidator, self).validate(document)
def notebook_path_to_dependencies(path_to_notebook,
remap=True,
custom_namespaces=None):
"""Helper function that turns a jupyter notebook into a list of dependencies
Parameters
----------
path_to_notebook : str
remap : bool, optional
Normalize the import names to be synonymous with their conda/pip names
custom_namespaces : list of str ot None, optional
If not None, then resulting package outputs will list everying under these
namespaces (e.g., for packages foo.bar and foo.baz, the outputs are foo.bar
and foo.baz instead of foo if custom_namespaces=["foo"]).
Returns
-------
dict
Dict of dependencies keyed on
- 'builtin' - libraries built in to python
- 'required' - libraries that are found at the top level of your modules
- 'questionable' - libraries that are found inside try/except blocks
- 'relative' - libraries that are relative imports
Examples
--------
>>> depfinder.notebook_path_to_dependencies('depfinder_usage.ipynb')
{'builtin': ['os', 'pprint'], 'required': ['depfinder']}
"""
try:
from IPython.core.inputsplitter import IPythonInputSplitter
transform = IPythonInputSplitter(
line_input_checker=False).transform_cell
except:
transform = lambda code: code
nb = json.load(io.open(path_to_notebook, encoding='utf8'))
codeblocks = [
''.join(cell['source']) for cell in nb['cells']
if cell['cell_type'] == 'code'
]
all_deps = defaultdict(set)
for codeblock in codeblocks:
codeblock = transform(codeblock)
# TODO this may fail on py2/py3 syntax when running in the other runtime.
# May want to consider updating some error handling around that case.
# Will wait until that use case surfaces before modifying
deps_dict = get_imported_libs(
codeblock, custom_namespaces=custom_namespaces).describe()
for k, v in deps_dict.items():
all_deps[k].update(v)
all_deps = {k: sorted(list(v)) for k, v in all_deps.items()}
if remap:
return sanitize_deps(all_deps)
return all_deps
def ipython2python(code):
"""Transform IPython syntax to pure Python syntax
Parameters
----------
code : str
IPython code, to be transformed to pure Python
"""
try:
from IPython.core.inputsplitter import IPythonInputSplitter
except ImportError:
warnings.warn(
"IPython is needed to transform IPython syntax to pure Python."
" Install ipython if you need this functionality.")
return code
else:
isp = IPythonInputSplitter(line_input_checker=False)
return isp.transform_cell(code)
def ipython2python(code):
"""Transform IPython syntax to pure Python syntax
Parameters
----------
code : str
IPython code, to be transformed to pure Python
"""
try:
from IPython.core.inputsplitter import IPythonInputSplitter
except ImportError:
warnings.warn(
"IPython is needed to transform IPython syntax to pure Python."
" Install ipython if you need this functionality."
)
return code
else:
isp = IPythonInputSplitter(line_input_checker=False)
return isp.transform_cell(code)
def transform_cell(code, indent_only=False):
"""Transform ipython code to python code."""
number_empty_lines = count_leading_empty_lines(code)
if indent_only:
# Not implemented for PY2
if LooseVersion(ipy_version) < LooseVersion('7.0.0'):
return code
if not code.endswith('\n'):
code += '\n' # Ensure the cell has a trailing newline
lines = code.splitlines(keepends=True)
lines = leading_indent(leading_empty_lines(lines))
code = ''.join(lines)
else:
if LooseVersion(ipy_version) < LooseVersion('7.0.0'):
tm = IPythonInputSplitter()
return tm.transform_cell(code)
else:
tm = TransformerManager()
code = tm.transform_cell(code)
return '\n' * number_empty_lines + code
def execute(code):
isp = IPythonInputSplitter()
def do_execute():
#print("CELL", cell )
result = kernel.shell.run_cell(cell, False)
if result.error_before_exec is not None:
err = result.error_before_exec
else:
err = result.error_in_exec
if not result.success:
for tb in kernel.shell._last_traceback:
print(tb)
return False
return True
for line in code.splitlines():
if isp.push_accepts_more():
isp.push(line.strip("\n"))
continue
cell = isp.source_reset()
if not do_execute():
return False
cell = isp.source_reset()
if len(cell):
return do_execute()
else:
return True
def ipython2python(code):
isp = IPythonInputSplitter()
newcode = ""
for line in code.splitlines():
if isp.push_accepts_more():
isp.push(line.strip("\n"))
continue
cell = isp.source_reset()
if cell.startswith("get_ipython().run_"):
cell = "_ = " + cell
newcode += cell + "\n"
isp.push(line.strip("\n"))
cell = isp.source_reset()
if len(cell):
if cell.startswith("get_ipython().run_"):
cell = "_ = " + cell
newcode += cell
return newcode
def __init__(self, *args, **kwargs):
# Initialization based on: from IPython.testing.globalipapp import start_ipython
self._curr_exec_line = 0
# Store certain global objects that IPython modifies
_displayhook = sys.displayhook
_excepthook = sys.excepthook
# Create and initialize our IPython instance.
shell = TerminalInteractiveShell.instance()
# Create an intput splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
shell.showtraceback = _showtraceback
# IPython is ready, now clean up some global state...
# Deactivate the various python system hooks added by ipython for
# interactive convenience so we don't confuse the doctest system
sys.displayhook = _displayhook
sys.excepthook = _excepthook
# So that ipython magics and aliases can be doctested (they work by making
# a call into a global _ip object). Also make the top-level get_ipython
# now return this without recursively calling here again.
get_ipython = shell.get_ipython
try:
import __builtin__
except:
import builtins as __builtin__
__builtin__._ip = shell
__builtin__.get_ipython = get_ipython
# We want to print to stdout/err as usual.
io.stdout = original_stdout
io.stderr = original_stderr
self.ipython = shell
def __init__(self, pydev_host, pydev_client_port, *args, **kwarg):
# Create and initialize our IPython instance.
self.ipython = PyDevTerminalInteractiveShell.instance()
# Back channel to PyDev to open editors (in the future other
# info may go back this way. This is the same channel that is
# used to get stdin, see StdIn in pydev_console_utils)
self.ipython.set_hook('editor', create_editor_hook(pydev_host, pydev_client_port))
# Create an input splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
# Display the IPython banner, this has version info and
# help info
self.ipython.show_banner()
def notebook_path_to_dependencies(path_to_notebook, remap=True):
"""Helper function that turns a jupyter notebook into a list of dependencies
Parameters
----------
path_to_notebook : str
remap : bool, optional
Normalize the import names to be synonymous with their conda/pip names
Returns
-------
dict
Dict of dependencies keyed on
- 'builtin' - libraries built in to python
- 'required' - libraries that are found at the top level of your modules
- 'questionable' - libraries that are found inside try/except blocks
- 'relative' - libraries that are relative imports
Examples
--------
>>> depfinder.notebook_path_to_dependencies('depfinder_usage.ipynb')
{'builtin': ['os', 'pprint'], 'required': ['depfinder']}
"""
try:
from IPython.core.inputsplitter import IPythonInputSplitter
transform = IPythonInputSplitter(line_input_checker=False).transform_cell
except:
transform = lambda code: code
nb = json.load(io.open(path_to_notebook, encoding='utf8'))
codeblocks = [''.join(cell['source']) for cell in nb['cells']
if cell['cell_type'] == 'code']
all_deps = defaultdict(set)
for codeblock in codeblocks:
codeblock = transform(codeblock)
deps_dict = get_imported_libs(codeblock).describe()
for k, v in deps_dict.items():
all_deps[k].update(v)
all_deps = {k: sorted(list(v)) for k, v in all_deps.items()}
if remap:
return sanitize_deps(all_deps)
return all_deps
def __init__(self, pydev_host, pydev_client_port, exec_queue, *args, **kwarg):
self.exec_queue = exec_queue
self._curr_exec_line = 0
# Store certain global objects that IPython modifies
_displayhook = sys.displayhook
_excepthook = sys.excepthook
# Create and initialize our IPython instance.
self.ipython = PyDevTerminalInteractiveShell.instance()
# Back channel to PyDev to open editors (in the future other
# info may go back this way. This is the same channel that is
# used to get stdin, see StdIn in pydev_console_utils)
self.ipython.set_hook('editor', create_editor_hook(pydev_host, pydev_client_port))
# Create an input splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
# Display the IPython banner, this has version info and
# help info
self.ipython.show_banner()
# IPython is ready, now clean up some global state...
# Deactivate the various python system hooks added by ipython for
# interactive convenience so we don't confuse the doctest system
sys.displayhook = _displayhook
sys.excepthook = _excepthook
# So that ipython magics and aliases can be doctested (they work by making
# a call into a global _ip object). Also make the top-level get_ipython
# now return this without recursively calling here again.
try:
import __builtin__
except:
import builtins as __builtin__
__builtin__._ip = self.ipython
__builtin__.get_ipython = self.ipython.get_ipython
# We want to print to stdout/err as usual.
io.stdout = original_stdout
io.stderr = original_stderr
def __init__(self, *args, **kwargs):
# Initialization based on: from IPython.testing.globalipapp import start_ipython
self._curr_exec_line = 0
# Store certain global objects that IPython modifies
_displayhook = sys.displayhook
_excepthook = sys.excepthook
# Create and initialize our IPython instance.
shell = TerminalInteractiveShell.instance()
# Create an intput splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
shell.showtraceback = _showtraceback
# IPython is ready, now clean up some global state...
# Deactivate the various python system hooks added by ipython for
# interactive convenience so we don't confuse the doctest system
sys.displayhook = _displayhook
sys.excepthook = _excepthook
# So that ipython magics and aliases can be doctested (they work by making
# a call into a global _ip object). Also make the top-level get_ipython
# now return this without recursively calling here again.
get_ipython = shell.get_ipython
try:
import __builtin__
except:
import builtins as __builtin__
__builtin__._ip = shell
__builtin__.get_ipython = get_ipython
# We want to print to stdout/err as usual.
io.stdout = original_stdout
io.stderr = original_stderr
self.ipython = shell
class PyDevFrontEnd:
def __init__(self, pydev_host, pydev_client_port, *args, **kwarg):
# Create and initialize our IPython instance.
self.ipython = PyDevTerminalInteractiveShell.instance()
# Back channel to PyDev to open editors (in the future other
# info may go back this way. This is the same channel that is
# used to get stdin, see StdIn in pydev_console_utils)
self.ipython.set_hook(
'editor', create_editor_hook(pydev_host, pydev_client_port))
# Create an input splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
# Display the IPython banner, this has version info and
# help info
self.ipython.show_banner()
def complete(self, string):
return self.ipython.complete(None, line=string)
def getCompletions(self, text, act_tok):
# Get completions from IPython and from PyDev and merge the results
# IPython only gives context free list of completions, while PyDev
# gives detailed information about completions.
try:
TYPE_IPYTHON = '11'
TYPE_IPYTHON_MAGIC = '12'
_line, ipython_completions = self.complete(text)
from _pydev_completer import Completer
completer = Completer(self.getNamespace(), None)
ret = completer.complete(act_tok)
append = ret.append
ip = self.ipython
pydev_completions = set([f[0] for f in ret])
for ipython_completion in ipython_completions:
if ipython_completion not in pydev_completions:
pydev_completions.add(ipython_completion)
inf = ip.object_inspect(ipython_completion)
if inf['type_name'] == 'Magic function':
pydev_type = TYPE_IPYTHON_MAGIC
else:
pydev_type = TYPE_IPYTHON
pydev_doc = inf['docstring']
if pydev_doc is None:
pydev_doc = ''
append((ipython_completion, pydev_doc, '', pydev_type))
return ret
except:
import traceback
traceback.print_exc()
return []
def getNamespace(self):
return self.ipython.user_ns
def addExec(self, line):
self.input_splitter.push(line)
if not self.input_splitter.push_accepts_more():
self.ipython.run_cell(self.input_splitter.source_reset(),
store_history=True)
return False
else:
return True
def module_from_spec(spec):
return _SpecMethods(spec).create()
def _init_module_attrs(spec, module):
return _SpecMethods(spec).init_module_attrs(module)
try:
import IPython
from IPython.core.inputsplitter import IPythonInputSplitter
dedent = IPythonInputSplitter(
line_input_checker=False,
physical_line_transforms=[
IPython.core.inputsplitter.leading_indent(),
IPython.core.inputsplitter.ipy_prompt(),
IPython.core.inputsplitter.cellmagic(end_on_blank_line=False),
],
).transform_cell
except:
from textwrap import dedent
__all__ = "Notebook", "reload"
class FinderContextManager:
"""
FinderContextManager is the base class for the notebook loader. It provides
a context manager that replaces `FileFinder` in the `sys.path_hooks` to include
an instance of the class in the python findering system.
def __init__(self, shell):
super(TerminalMagics, self).__init__(shell)
self.input_splitter = IPythonInputSplitter()
def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False, cwd=None, test_dir=None, seed=None):
"""
Executes a notebook and returns the global environment that results from execution
Execute notebook & return the global environment that results from execution. If ``ignore_errors``
is ``True``, exceptions are swallowed. ``secret`` contains random digits so ``check_results`` and
``check`` are not easily modifiable. ``nb`` is passed in as a dictionary that's a parsed notebook
Args:
nb (``dict``): JSON representation of a notebook
secret (``str``, optional): randomly generated integer used to rebind check function
initial_env (``str``, optional): name of initial environment
ignore_errors (``bool``, optional): whether exceptions should be ignored
cwd (``str``, optional): working directory of execution to be appended to ``sys.path`` in
grading environment
test_dir (``str``, optional): path to directory of tests in grading environment
seed (``int``, optional): random seed for intercell seeding
Results:
``dict``: global environment resulting from executing all code of the input notebook
"""
with hide_outputs():
if initial_env:
global_env = initial_env.copy()
else:
global_env = {}
# add display from IPython
global_env["display"] = display
source = ""
# if gradescope:
# source = "import sys\nsys.path.append(\"/autograder/submission\")\n"
# el
if cwd:
source = f"import sys\nsys.path.append(r\"{cwd}\")\n"
exec(source, global_env)
if seed is not None:
# source += "import numpy as np\nimport random\n"
import numpy as np
import random
global_env["np"] = np
global_env["random"] = random
# Before rewriting AST, find cells of code that generate errors.
# One round of execution is done beforehand to mimic the Jupyter notebook style of running
# (e.g. code runs up to the point of execution).
# The reason this is workaround is introduced is because once the
# source code is parsed into an AST, there is no sense of local cells
for cell in nb['cells']:
if cell['cell_type'] == 'code':
# transform the input to executable Python
# FIXME: use appropriate IPython functions here
isp = IPythonInputSplitter(line_input_checker=False)
try:
code_lines = []
cell_source_lines = cell['source']
source_is_str_bool = False
if isinstance(cell_source_lines, str):
source_is_str_bool = True
cell_source_lines = cell_source_lines.split('\n')
for line in cell_source_lines:
# Filter out ipython magic commands
# Filter out interact widget
if not line.startswith('%'):
if "interact(" not in line and not re.search(r"otter\.Notebook\(.*?\)", line):
code_lines.append(line)
if source_is_str_bool:
code_lines.append('\n')
elif re.search(r"otter\.Notebook\(.*?\)", line):
# TODO: move this check into CheckCallWrapper
# if gradescope:
# line = re.sub(r"otter\.Notebook\(.*?\)", "otter.Notebook(\"/autograder/submission/tests\")", line)
# el
if test_dir:
line = re.sub(r"otter\.Notebook\(.*?\)", f"otter.Notebook(\"{test_dir}\")", line)
else:
line = re.sub(r"otter\.Notebook\(.*?\)", "otter.Notebook(\"/home/tests\")", line)
code_lines.append(line)
if source_is_str_bool:
code_lines.append('\n')
if seed is not None:
cell_source = "np.random.seed({})\nrandom.seed({})\n".format(seed, seed) + isp.transform_cell(''.join(code_lines))
else:
cell_source = isp.transform_cell(''.join(code_lines))
# patch otter.Notebook.export so that we don't create PDFs in notebooks
# TODO: move this patch into CheckCallWrapper
m = mock.mock_open()
with mock.patch('otter.Notebook.export', m), mock.patch("otter.Notebook._log_event", m):
exec(cell_source, global_env)
source += cell_source
except:
if not ignore_errors:
raise
tree = ast.parse(source)
# # CODE BELOW COMMENTED OUT BECAUSE the only check function is within the Notebook class
# if find_check_assignment(tree) or find_check_definition(tree):
# # an empty global_env will fail all the tests
# return global_env
# wrap check(..) calls into a check_results_X.append(check(..))
transformer = CheckCallWrapper(secret)
tree = transformer.visit(tree)
ast.fix_missing_locations(tree)
cleaned_source = compile(tree, filename="nb-ast", mode="exec")
try:
with open(os.devnull, 'w') as f, redirect_stdout(f), redirect_stderr(f):
# patch otter.Notebook.export so that we don't create PDFs in notebooks
m = mock.mock_open()
with mock.patch('otter.Notebook.export', m), mock.patch("otter.Notebook._log_event", m):
exec(cleaned_source, global_env)
except:
if not ignore_errors:
raise
return global_env
class IPythonWidget(FrontendWidget):
""" A FrontendWidget for an IPython kernel.
"""
# If set, the 'custom_edit_requested(str, int)' signal will be emitted when
# an editor is needed for a file. This overrides 'editor' and 'editor_line'
# settings.
custom_edit = Bool(False)
custom_edit_requested = QtCore.Signal(object, object)
editor = Unicode(default_editor,
config=True,
help="""
A command for invoking a system text editor. If the string contains a
{filename} format specifier, it will be used. Otherwise, the filename
will be appended to the end the command.
""")
editor_line = Unicode(config=True,
help="""
The editor command to use when a specific line number is requested. The
string should contain two format specifiers: {line} and {filename}. If
this parameter is not specified, the line number option to the %edit
magic will be ignored.
""")
style_sheet = Unicode(config=True,
help="""
A CSS stylesheet. The stylesheet can contain classes for:
1. Qt: QPlainTextEdit, QFrame, QWidget, etc
2. Pygments: .c, .k, .o, etc. (see PygmentsHighlighter)
3. IPython: .error, .in-prompt, .out-prompt, etc
""")
syntax_style = Unicode(config=True,
help="""
If not empty, use this Pygments style for syntax highlighting.
Otherwise, the style sheet is queried for Pygments style
information.
""")
# Prompts.
in_prompt = Unicode(default_in_prompt, config=True)
out_prompt = Unicode(default_out_prompt, config=True)
input_sep = Unicode(default_input_sep, config=True)
output_sep = Unicode(default_output_sep, config=True)
output_sep2 = Unicode(default_output_sep2, config=True)
# FrontendWidget protected class variables.
_input_splitter_class = IPythonInputSplitter
_prompt_transformer = IPythonInputSplitter(
physical_line_transforms=[ipy_prompt()],
logical_line_transforms=[],
python_line_transforms=[],
)
# IPythonWidget protected class variables.
_PromptBlock = namedtuple('_PromptBlock', ['block', 'length', 'number'])
_payload_source_edit = 'edit_magic'
_payload_source_exit = 'ask_exit'
_payload_source_next_input = 'set_next_input'
_payload_source_page = 'page'
_retrying_history_request = False
#---------------------------------------------------------------------------
# 'object' interface
#---------------------------------------------------------------------------
def __init__(self, *args, **kw):
super(IPythonWidget, self).__init__(*args, **kw)
# IPythonWidget protected variables.
self._payload_handlers = {
self._payload_source_edit: self._handle_payload_edit,
self._payload_source_exit: self._handle_payload_exit,
self._payload_source_page: self._handle_payload_page,
self._payload_source_next_input: self._handle_payload_next_input
}
self._previous_prompt_obj = None
self._keep_kernel_on_exit = None
# Initialize widget styling.
if self.style_sheet:
self._style_sheet_changed()
self._syntax_style_changed()
else:
self.set_default_style()
#---------------------------------------------------------------------------
# 'BaseFrontendMixin' abstract interface
#---------------------------------------------------------------------------
def _handle_complete_reply(self, rep):
""" Reimplemented to support IPython's improved completion machinery.
"""
self.log.debug("complete: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('complete')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
matches = rep['content']['matches']
text = rep['content']['matched_text']
offset = len(text)
# Clean up matches with period and path separators if the matched
# text has not been transformed. This is done by truncating all
# but the last component and then suitably decreasing the offset
# between the current cursor position and the start of completion.
if len(matches) > 1 and matches[0][:offset] == text:
parts = re.split(r'[./\\]', text)
sep_count = len(parts) - 1
if sep_count:
chop_length = sum(map(len, parts[:sep_count])) + sep_count
matches = [match[chop_length:] for match in matches]
offset -= chop_length
# Move the cursor to the start of the match and complete.
cursor.movePosition(QtGui.QTextCursor.Left, n=offset)
self._complete_with_items(cursor, matches)
def _handle_execute_reply(self, msg):
""" Reimplemented to support prompt requests.
"""
msg_id = msg['parent_header'].get('msg_id')
info = self._request_info['execute'].get(msg_id)
if info and info.kind == 'prompt':
number = msg['content']['execution_count'] + 1
self._show_interpreter_prompt(number)
self._request_info['execute'].pop(msg_id)
else:
super(IPythonWidget, self)._handle_execute_reply(msg)
def _handle_history_reply(self, msg):
""" Implemented to handle history tail replies, which are only supported
by the IPython kernel.
"""
content = msg['content']
if 'history' not in content:
self.log.error("History request failed: %r" % content)
if content.get('status', '') == 'aborted' and \
not self._retrying_history_request:
# a *different* action caused this request to be aborted, so
# we should try again.
self.log.error("Retrying aborted history request")
# prevent multiple retries of aborted requests:
self._retrying_history_request = True
# wait out the kernel's queue flush, which is currently timed at 0.1s
time.sleep(0.25)
self.kernel_client.shell_channel.history(
hist_access_type='tail', n=1000)
else:
self._retrying_history_request = False
return
# reset retry flag
self._retrying_history_request = False
history_items = content['history']
self.log.debug("Received history reply with %i entries",
len(history_items))
items = []
last_cell = u""
for _, _, cell in history_items:
cell = cell.rstrip()
if cell != last_cell:
items.append(cell)
last_cell = cell
self._set_history(items)
def _handle_pyout(self, msg):
""" Reimplemented for IPython-style "display hook".
"""
self.log.debug("pyout: %s", msg.get('content', ''))
if not self._hidden and self._is_from_this_session(msg):
content = msg['content']
prompt_number = content.get('execution_count', 0)
data = content['data']
if 'text/html' in data:
self._append_plain_text(self.output_sep, True)
self._append_html(self._make_out_prompt(prompt_number), True)
html = data['text/html']
self._append_plain_text('\n', True)
self._append_html(html + self.output_sep2, True)
elif 'text/plain' in data:
self._append_plain_text(self.output_sep, True)
self._append_html(self._make_out_prompt(prompt_number), True)
text = data['text/plain']
# If the repr is multiline, make sure we start on a new line,
# so that its lines are aligned.
if "\n" in text and not self.output_sep.endswith("\n"):
self._append_plain_text('\n', True)
self._append_plain_text(text + self.output_sep2, True)
def _handle_display_data(self, msg):
""" The base handler for the ``display_data`` message.
"""
self.log.debug("display: %s", msg.get('content', ''))
# For now, we don't display data from other frontends, but we
# eventually will as this allows all frontends to monitor the display
# data. But we need to figure out how to handle this in the GUI.
if not self._hidden and self._is_from_this_session(msg):
source = msg['content']['source']
data = msg['content']['data']
metadata = msg['content']['metadata']
# In the regular IPythonWidget, we simply print the plain text
# representation.
if 'text/html' in data:
html = data['text/html']
self._append_html(html, True)
elif 'text/plain' in data:
text = data['text/plain']
self._append_plain_text(text, True)
# This newline seems to be needed for text and html output.
self._append_plain_text(u'\n', True)
def _started_channels(self):
"""Reimplemented to make a history request and load %guiref."""
super(IPythonWidget, self)._started_channels()
self._load_guiref_magic()
self.kernel_client.shell_channel.history(hist_access_type='tail',
n=1000)
def _started_kernel(self):
"""Load %guiref when the kernel starts (if channels are also started).
Principally triggered by kernel restart.
"""
if self.kernel_client.shell_channel is not None:
self._load_guiref_magic()
def _load_guiref_magic(self):
"""Load %guiref magic."""
self.kernel_client.shell_channel.execute('\n'.join([
"try:",
" _usage",
"except:",
" from IPython.core import usage as _usage",
" get_ipython().register_magic_function(_usage.page_guiref, 'line', 'guiref')",
" del _usage",
]),
silent=True)
#---------------------------------------------------------------------------
# 'ConsoleWidget' public interface
#---------------------------------------------------------------------------
#---------------------------------------------------------------------------
# 'FrontendWidget' public interface
#---------------------------------------------------------------------------
def execute_file(self, path, hidden=False):
""" Reimplemented to use the 'run' magic.
"""
# Use forward slashes on Windows to avoid escaping each separator.
if sys.platform == 'win32':
path = os.path.normpath(path).replace('\\', '/')
# Perhaps we should not be using %run directly, but while we
# are, it is necessary to quote or escape filenames containing spaces
# or quotes.
# In earlier code here, to minimize escaping, we sometimes quoted the
# filename with single quotes. But to do this, this code must be
# platform-aware, because run uses shlex rather than python string
# parsing, so that:
# * In Win: single quotes can be used in the filename without quoting,
# and we cannot use single quotes to quote the filename.
# * In *nix: we can escape double quotes in a double quoted filename,
# but can't escape single quotes in a single quoted filename.
# So to keep this code non-platform-specific and simple, we now only
# use double quotes to quote filenames, and escape when needed:
if ' ' in path or "'" in path or '"' in path:
path = '"%s"' % path.replace('"', '\\"')
self.execute('%%run %s' % path, hidden=hidden)
#---------------------------------------------------------------------------
# 'FrontendWidget' protected interface
#---------------------------------------------------------------------------
def _complete(self):
""" Reimplemented to support IPython's improved completion machinery.
"""
# We let the kernel split the input line, so we *always* send an empty
# text field. Readline-based frontends do get a real text field which
# they can use.
text = ''
# Send the completion request to the kernel
msg_id = self.kernel_client.shell_channel.complete(
text, # text
self._get_input_buffer_cursor_line(), # line
self._get_input_buffer_cursor_column(), # cursor_pos
self.input_buffer) # block
pos = self._get_cursor().position()
info = self._CompletionRequest(msg_id, pos)
self._request_info['complete'] = info
def _process_execute_error(self, msg):
""" Reimplemented for IPython-style traceback formatting.
"""
content = msg['content']
traceback = '\n'.join(content['traceback']) + '\n'
if False:
# FIXME: For now, tracebacks come as plain text, so we can't use
# the html renderer yet. Once we refactor ultratb to produce
# properly styled tracebacks, this branch should be the default
traceback = traceback.replace(' ', ' ')
traceback = traceback.replace('\n', '<br/>')
ename = content['ename']
ename_styled = '<span class="error">%s</span>' % ename
traceback = traceback.replace(ename, ename_styled)
self._append_html(traceback)
else:
# This is the fallback for now, using plain text with ansi escapes
self._append_plain_text(traceback)
def _process_execute_payload(self, item):
""" Reimplemented to dispatch payloads to handler methods.
"""
handler = self._payload_handlers.get(item['source'])
if handler is None:
# We have no handler for this type of payload, simply ignore it
return False
else:
handler(item)
return True
def _show_interpreter_prompt(self, number=None):
""" Reimplemented for IPython-style prompts.
"""
# If a number was not specified, make a prompt number request.
if number is None:
msg_id = self.kernel_client.shell_channel.execute('', silent=True)
info = self._ExecutionRequest(msg_id, 'prompt')
self._request_info['execute'][msg_id] = info
return
# Show a new prompt and save information about it so that it can be
# updated later if the prompt number turns out to be wrong.
self._prompt_sep = self.input_sep
self._show_prompt(self._make_in_prompt(number), html=True)
block = self._control.document().lastBlock()
length = len(self._prompt)
self._previous_prompt_obj = self._PromptBlock(block, length, number)
# Update continuation prompt to reflect (possibly) new prompt length.
self._set_continuation_prompt(self._make_continuation_prompt(
self._prompt),
html=True)
def _show_interpreter_prompt_for_reply(self, msg):
""" Reimplemented for IPython-style prompts.
"""
# Update the old prompt number if necessary.
content = msg['content']
# abort replies do not have any keys:
if content['status'] == 'aborted':
if self._previous_prompt_obj:
previous_prompt_number = self._previous_prompt_obj.number
else:
previous_prompt_number = 0
else:
previous_prompt_number = content['execution_count']
if self._previous_prompt_obj and \
self._previous_prompt_obj.number != previous_prompt_number:
block = self._previous_prompt_obj.block
# Make sure the prompt block has not been erased.
if block.isValid() and block.text():
# Remove the old prompt and insert a new prompt.
cursor = QtGui.QTextCursor(block)
cursor.movePosition(QtGui.QTextCursor.Right,
QtGui.QTextCursor.KeepAnchor,
self._previous_prompt_obj.length)
prompt = self._make_in_prompt(previous_prompt_number)
self._prompt = self._insert_html_fetching_plain_text(
cursor, prompt)
# When the HTML is inserted, Qt blows away the syntax
# highlighting for the line, so we need to rehighlight it.
self._highlighter.rehighlightBlock(cursor.block())
self._previous_prompt_obj = None
# Show a new prompt with the kernel's estimated prompt number.
self._show_interpreter_prompt(previous_prompt_number + 1)
#---------------------------------------------------------------------------
# 'IPythonWidget' interface
#---------------------------------------------------------------------------
def set_default_style(self, colors='lightbg'):
""" Sets the widget style to the class defaults.
Parameters
----------
colors : str, optional (default lightbg)
Whether to use the default IPython light background or dark
background or B&W style.
"""
colors = colors.lower()
if colors == 'lightbg':
self.style_sheet = styles.default_light_style_sheet
self.syntax_style = styles.default_light_syntax_style
elif colors == 'linux':
self.style_sheet = styles.default_dark_style_sheet
self.syntax_style = styles.default_dark_syntax_style
elif colors == 'nocolor':
self.style_sheet = styles.default_bw_style_sheet
self.syntax_style = styles.default_bw_syntax_style
else:
raise KeyError("No such color scheme: %s" % colors)
#---------------------------------------------------------------------------
# 'IPythonWidget' protected interface
#---------------------------------------------------------------------------
def _edit(self, filename, line=None):
""" Opens a Python script for editing.
Parameters
----------
filename : str
A path to a local system file.
line : int, optional
A line of interest in the file.
"""
if self.custom_edit:
self.custom_edit_requested.emit(filename, line)
elif not self.editor:
self._append_plain_text(
'No default editor available.\n'
'Specify a GUI text editor in the `IPythonWidget.editor` '
'configurable to enable the %edit magic')
else:
try:
filename = '"%s"' % filename
if line and self.editor_line:
command = self.editor_line.format(filename=filename,
line=line)
else:
try:
command = self.editor.format()
except KeyError:
command = self.editor.format(filename=filename)
else:
command += ' ' + filename
except KeyError:
self._append_plain_text('Invalid editor command.\n')
else:
try:
Popen(command, shell=True)
except OSError:
msg = 'Opening editor with command "%s" failed.\n'
self._append_plain_text(msg % command)
def _make_in_prompt(self, number):
""" Given a prompt number, returns an HTML In prompt.
"""
try:
body = self.in_prompt % number
except TypeError:
# allow in_prompt to leave out number, e.g. '>>> '
body = self.in_prompt
return '<span class="in-prompt">%s</span>' % body
def _make_continuation_prompt(self, prompt):
""" Given a plain text version of an In prompt, returns an HTML
continuation prompt.
"""
end_chars = '...: '
space_count = len(prompt.lstrip('\n')) - len(end_chars)
body = ' ' * space_count + end_chars
return '<span class="in-prompt">%s</span>' % body
def _make_out_prompt(self, number):
""" Given a prompt number, returns an HTML Out prompt.
"""
body = self.out_prompt % number
return '<span class="out-prompt">%s</span>' % body
#------ Payload handlers --------------------------------------------------
# Payload handlers with a generic interface: each takes the opaque payload
# dict, unpacks it and calls the underlying functions with the necessary
# arguments.
def _handle_payload_edit(self, item):
self._edit(item['filename'], item['line_number'])
def _handle_payload_exit(self, item):
self._keep_kernel_on_exit = item['keepkernel']
self.exit_requested.emit(self)
def _handle_payload_next_input(self, item):
self.input_buffer = item['text']
def _handle_payload_page(self, item):
# Since the plain text widget supports only a very small subset of HTML
# and we have no control over the HTML source, we only page HTML
# payloads in the rich text widget.
if item['html'] and self.kind == 'rich':
self._page(item['html'], html=True)
else:
self._page(item['text'], html=False)
#------ Trait change handlers --------------------------------------------
def _style_sheet_changed(self):
""" Set the style sheets of the underlying widgets.
"""
self.setStyleSheet(self.style_sheet)
if self._control is not None:
self._control.document().setDefaultStyleSheet(self.style_sheet)
bg_color = self._control.palette().window().color()
self._ansi_processor.set_background_color(bg_color)
if self._page_control is not None:
self._page_control.document().setDefaultStyleSheet(
self.style_sheet)
def _syntax_style_changed(self):
""" Set the style for the syntax highlighter.
"""
if self._highlighter is None:
# ignore premature calls
return
if self.syntax_style:
self._highlighter.set_style(self.syntax_style)
else:
self._highlighter.set_style_sheet(self.style_sheet)
#------ Trait default initializers -----------------------------------------
def _banner_default(self):
from IPython.core.usage import default_gui_banner
return default_gui_banner
def execute_notebook(nb,
secret='secret',
initial_env=None,
ignore_errors=False,
script=False):
"""
Execute notebook & return the global environment that results from execution.
TODO: write a note about the injection of check_results
If ignore_errors is True, exceptions are swallowed.
secret contains random digits so check_results and check are not easily modifiable
nb is passed in as a dictionary that's a parsed ipynb file
"""
with hide_outputs():
if initial_env:
global_env = initial_env.copy()
else:
global_env = {}
source = ""
if script:
try:
exec(nb, global_env)
source += nb
except:
if not ignore_errors:
raise
else:
# Before rewriting AST, find cells of code that generate errors.
# One round of execution is done beforehand to mimic the Jupyter notebook style of running
# (e.g. code runs up to the point of execution).
# The reason this is workaround is introduced is because once the
# source code is parsed into an AST, there is no sense of local cells
for cell in nb['cells']:
if cell['cell_type'] == 'code':
# transform the input to executable Python
# FIXME: use appropriate IPython functions here
isp = IPythonInputSplitter(line_input_checker=False)
try:
code_lines = []
cell_source_lines = cell['source']
source_is_str_bool = False
if isinstance(cell_source_lines, str):
source_is_str_bool = True
cell_source_lines = cell_source_lines.split('\n')
for line in cell_source_lines:
# Filter out ipython magic commands
# Filter out interact widget
if not line.startswith('%'):
if "interact(" not in line:
code_lines.append(line)
if source_is_str_bool:
code_lines.append('\n')
cell_source = isp.transform_cell(''.join(code_lines))
exec(cell_source, global_env)
source += cell_source
except:
if not ignore_errors:
raise
tree = ast.parse(source)
if find_check_assignment(tree) or find_check_definition(tree):
# an empty global_env will fail all the tests
return global_env
# wrap check(..) calls into a check_results_X.append(check(..))
transformer = CheckCallWrapper(secret)
tree = transformer.visit(tree)
ast.fix_missing_locations(tree)
cleaned_source = compile(tree, filename="nb-ast", mode="exec")
try:
with open(os.devnull,
'w') as f, redirect_stdout(f), redirect_stderr(f):
exec(cleaned_source, global_env)
except:
if not ignore_errors:
raise
return global_env
class PyDevFrontEnd:
def __init__(self, pydev_host, pydev_client_port, *args, **kwarg):
# Create and initialize our IPython instance.
self.ipython = PyDevTerminalInteractiveShell.instance()
# Back channel to PyDev to open editors (in the future other
# info may go back this way. This is the same channel that is
# used to get stdin, see StdIn in pydev_console_utils)
self.ipython.set_hook('editor', create_editor_hook(pydev_host, pydev_client_port))
# Create an input splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
# Display the IPython banner, this has version info and
# help info
self.ipython.show_banner()
def complete(self, string):
return self.ipython.complete(None, line=string)
def getCompletions(self, text, act_tok):
# Get completions from IPython and from PyDev and merge the results
# IPython only gives context free list of completions, while PyDev
# gives detailed information about completions.
try:
TYPE_IPYTHON = '11'
TYPE_IPYTHON_MAGIC = '12'
_line, ipython_completions = self.complete(text)
from _pydev_completer import Completer
completer = Completer(self.getNamespace(), None)
ret = completer.complete(act_tok)
append = ret.append
ip = self.ipython
pydev_completions = set([f[0] for f in ret])
for ipython_completion in ipython_completions:
if ipython_completion not in pydev_completions:
pydev_completions.add(ipython_completion)
inf = ip.object_inspect(ipython_completion)
if inf['type_name'] == 'Magic function':
pydev_type = TYPE_IPYTHON_MAGIC
else:
pydev_type = TYPE_IPYTHON
pydev_doc = inf['docstring']
if pydev_doc is None:
pydev_doc = ''
append((ipython_completion, pydev_doc, '', pydev_type))
return ret
except:
import traceback;traceback.print_exc()
return []
def getNamespace(self):
return self.ipython.user_ns
def addExec(self, line):
self.input_splitter.push(line)
if not self.input_splitter.push_accepts_more():
self.ipython.run_cell(self.input_splitter.source_reset(), store_history=True)
return False
else:
return True
class PyDevFrontEnd:
def __init__(self, pydev_host, pydev_client_port, exec_queue, *args, **kwarg):
self.exec_queue = exec_queue
self._curr_exec_line = 0
# Store certain global objects that IPython modifies
_displayhook = sys.displayhook
_excepthook = sys.excepthook
# Create and initialize our IPython instance.
self.ipython = PyDevTerminalInteractiveShell.instance()
# Back channel to PyDev to open editors (in the future other
# info may go back this way. This is the same channel that is
# used to get stdin, see StdIn in pydev_console_utils)
self.ipython.set_hook('editor', create_editor_hook(pydev_host, pydev_client_port))
# Create an input splitter to handle input separation
self.input_splitter = IPythonInputSplitter()
# Display the IPython banner, this has version info and
# help info
self.ipython.show_banner()
# IPython is ready, now clean up some global state...
# Deactivate the various python system hooks added by ipython for
# interactive convenience so we don't confuse the doctest system
sys.displayhook = _displayhook
sys.excepthook = _excepthook
# So that ipython magics and aliases can be doctested (they work by making
# a call into a global _ip object). Also make the top-level get_ipython
# now return this without recursively calling here again.
try:
import __builtin__
except:
import builtins as __builtin__
__builtin__._ip = self.ipython
__builtin__.get_ipython = self.ipython.get_ipython
# We want to print to stdout/err as usual.
io.stdout = original_stdout
io.stderr = original_stderr
def complete(self, string):
return self.ipython.complete(None, line=string)
def getCompletions(self, text, act_tok):
# Get completions from IPython and from PyDev and merge the results
# IPython only gives context free list of completions, while PyDev
# gives detailed information about completions.
try:
TYPE_IPYTHON = '11'
TYPE_IPYTHON_MAGIC = '12'
_line, ipython_completions = self.complete(text)
from _pydev_completer import Completer
completer = Completer(self.getNamespace(), None)
ret = completer.complete(act_tok)
append = ret.append
ip = self.ipython
pydev_completions = set([f[0] for f in ret])
for ipython_completion in ipython_completions:
if ipython_completion not in pydev_completions:
pydev_completions.add(ipython_completion)
inf = ip.object_inspect(ipython_completion)
if inf['type_name'] == 'Magic function':
pydev_type = TYPE_IPYTHON_MAGIC
else:
pydev_type = TYPE_IPYTHON
pydev_doc = inf['docstring']
if pydev_doc is None:
pydev_doc = ''
append((ipython_completion, pydev_doc, '', pydev_type))
return ret
except:
import traceback;traceback.print_exc()
return []
def interrupt(self):
self.input_splitter.reset()
def getNamespace(self):
return self.ipython.user_ns
def addExec(self, line):
self.input_splitter.push(line)
if not self.input_splitter.push_accepts_more():
self.exec_queue.put(partial(self.ipython.run_cell, self.input_splitter.source_reset(), store_history=True))
return False
else:
return True
import secrets
import sys
import warnings
from abc import ABC
from ast import AST
from enum import Enum
from textwrap import dedent, indent
from typing import ClassVar, Mapping, Optional, Pattern, Sequence
with warnings.catch_warnings():
# see https://github.com/nbQA-dev/nbQA/issues/459
warnings.filterwarnings("ignore", category=DeprecationWarning)
from IPython.core.inputsplitter import IPythonInputSplitter
INPUT_SPLITTER = IPythonInputSplitter(line_input_checker=False)
COMMANDS_WITH_STRING_TOKEN = {"flake8"}
class IPythonMagicType(Enum):
"""Enumeration representing various types of IPython magics."""
SHELL = 0
HELP = 1
LINE = 2
CELL = 3
NO_MAGIC = 4
class MagicHandler(ABC):
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):
""" A Qt frontend for a generic Python kernel.
"""
# The text to show when the kernel is (re)started.
banner = Unicode(config=True)
# An option and corresponding signal for overriding the default kernel
# interrupt behavior.
custom_interrupt = Bool(False)
custom_interrupt_requested = QtCore.Signal()
# An option and corresponding signals for overriding the default kernel
# restart behavior.
custom_restart = Bool(False)
custom_restart_kernel_died = QtCore.Signal(float)
custom_restart_requested = QtCore.Signal()
# Whether to automatically show calltips on open-parentheses.
enable_calltips = Bool(
True,
config=True,
help="Whether to draw information calltips on open-parentheses.")
clear_on_kernel_restart = Bool(
True,
config=True,
help="Whether to clear the console when the kernel is restarted")
confirm_restart = Bool(
True,
config=True,
help="Whether to ask for user confirmation when restarting kernel")
lexer_class = DottedObjectName(config=True,
help="The pygments lexer class to use.")
def _lexer_class_changed(self, name, old, new):
lexer_class = import_item(new)
self.lexer = lexer_class()
def _lexer_class_default(self):
if py3compat.PY3:
return 'pygments.lexers.Python3Lexer'
else:
return 'pygments.lexers.PythonLexer'
lexer = Any()
def _lexer_default(self):
lexer_class = import_item(self.lexer_class)
return lexer_class()
# Emitted when a user visible 'execute_request' has been submitted to the
# kernel from the FrontendWidget. Contains the code to be executed.
executing = QtCore.Signal(object)
# Emitted when a user-visible 'execute_reply' has been received from the
# kernel and processed by the FrontendWidget. Contains the response message.
executed = QtCore.Signal(object)
# Emitted when an exit request has been received from the kernel.
exit_requested = QtCore.Signal(object)
# Protected class variables.
_prompt_transformer = IPythonInputSplitter(
physical_line_transforms=[classic_prompt()],
logical_line_transforms=[],
python_line_transforms=[],
)
_CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos'])
_CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos'])
_ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind'])
_input_splitter_class = InputSplitter
_local_kernel = False
_highlighter = Instance(FrontendHighlighter)
#---------------------------------------------------------------------------
# 'object' interface
#---------------------------------------------------------------------------
def __init__(self, *args, **kw):
super(FrontendWidget, self).__init__(*args, **kw)
# FIXME: remove this when PySide min version is updated past 1.0.7
# forcefully disable calltips if PySide is < 1.0.7, because they crash
if qt.QT_API == qt.QT_API_PYSIDE:
import PySide
if PySide.__version_info__ < (1, 0, 7):
self.log.warn(
"PySide %s < 1.0.7 detected, disabling calltips" %
PySide.__version__)
self.enable_calltips = False
# FrontendWidget protected variables.
self._bracket_matcher = BracketMatcher(self._control)
self._call_tip_widget = CallTipWidget(self._control)
self._completion_lexer = CompletionLexer(self.lexer)
self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None)
self._hidden = False
self._highlighter = FrontendHighlighter(self, lexer=self.lexer)
self._input_splitter = self._input_splitter_class()
self._kernel_manager = None
self._kernel_client = None
self._request_info = {}
self._request_info['execute'] = {}
self._callback_dict = {}
# Configure the ConsoleWidget.
self.tab_width = 4
self._set_continuation_prompt('... ')
# Configure the CallTipWidget.
self._call_tip_widget.setFont(self.font)
self.font_changed.connect(self._call_tip_widget.setFont)
# Configure actions.
action = self._copy_raw_action
key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C
action.setEnabled(False)
action.setShortcut(QtGui.QKeySequence(key))
action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut)
action.triggered.connect(self.copy_raw)
self.copy_available.connect(action.setEnabled)
self.addAction(action)
# Connect signal handlers.
document = self._control.document()
document.contentsChange.connect(self._document_contents_change)
# Set flag for whether we are connected via localhost.
self._local_kernel = kw.get('local_kernel',
FrontendWidget._local_kernel)
#---------------------------------------------------------------------------
# 'ConsoleWidget' public interface
#---------------------------------------------------------------------------
def copy(self):
""" Copy the currently selected text to the clipboard, removing prompts.
"""
if self._page_control is not None and self._page_control.hasFocus():
self._page_control.copy()
elif self._control.hasFocus():
text = self._control.textCursor().selection().toPlainText()
if text:
text = self._prompt_transformer.transform_cell(text)
QtGui.QApplication.clipboard().setText(text)
else:
self.log.debug("frontend widget : unknown copy target")
#---------------------------------------------------------------------------
# 'ConsoleWidget' abstract interface
#---------------------------------------------------------------------------
def _is_complete(self, source, interactive):
""" Returns whether 'source' can be completely processed and a new
prompt created. When triggered by an Enter/Return key press,
'interactive' is True; otherwise, it is False.
"""
self._input_splitter.reset()
complete = self._input_splitter.push(source)
if interactive:
complete = not self._input_splitter.push_accepts_more()
return complete
def _execute(self, source, hidden):
""" Execute 'source'. If 'hidden', do not show any output.
See parent class :meth:`execute` docstring for full details.
"""
msg_id = self.kernel_client.execute(source, hidden)
self._request_info['execute'][msg_id] = self._ExecutionRequest(
msg_id, 'user')
self._hidden = hidden
if not hidden:
self.executing.emit(source)
def _prompt_started_hook(self):
""" Called immediately after a new prompt is displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = True
def _prompt_finished_hook(self):
""" Called immediately after a prompt is finished, i.e. when some input
will be processed and a new prompt displayed.
"""
# Flush all state from the input splitter so the next round of
# reading input starts with a clean buffer.
self._input_splitter.reset()
if not self._reading:
self._highlighter.highlighting_on = False
def _tab_pressed(self):
""" Called when the tab key is pressed. Returns whether to continue
processing the event.
"""
# Perform tab completion if:
# 1) The cursor is in the input buffer.
# 2) There is a non-whitespace character before the cursor.
text = self._get_input_buffer_cursor_line()
if text is None:
return False
complete = bool(text[:self._get_input_buffer_cursor_column()].strip())
if complete:
self._complete()
return not complete
#---------------------------------------------------------------------------
# 'ConsoleWidget' protected interface
#---------------------------------------------------------------------------
def _context_menu_make(self, pos):
""" Reimplemented to add an action for raw copy.
"""
menu = super(FrontendWidget, self)._context_menu_make(pos)
for before_action in menu.actions():
if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \
QtGui.QKeySequence.ExactMatch:
menu.insertAction(before_action, self._copy_raw_action)
break
return menu
def request_interrupt_kernel(self):
if self._executing:
self.interrupt_kernel()
def request_restart_kernel(self):
message = 'Are you sure you want to restart the kernel?'
self.restart_kernel(message, now=False)
def _event_filter_console_keypress(self, event):
""" Reimplemented for execution interruption and smart backspace.
"""
key = event.key()
if self._control_key_down(event.modifiers(), include_command=False):
if key == QtCore.Qt.Key_C and self._executing:
self.request_interrupt_kernel()
return True
elif key == QtCore.Qt.Key_Period:
self.request_restart_kernel()
return True
elif not event.modifiers() & QtCore.Qt.AltModifier:
# Smart backspace: remove four characters in one backspace if:
# 1) everything left of the cursor is whitespace
# 2) the four characters immediately left of the cursor are spaces
if key == QtCore.Qt.Key_Backspace:
col = self._get_input_buffer_cursor_column()
cursor = self._control.textCursor()
if col > 3 and not cursor.hasSelection():
text = self._get_input_buffer_cursor_line()[:col]
if text.endswith(' ') and not text.strip():
cursor.movePosition(QtGui.QTextCursor.Left,
QtGui.QTextCursor.KeepAnchor, 4)
cursor.removeSelectedText()
return True
return super(FrontendWidget,
self)._event_filter_console_keypress(event)
def _insert_continuation_prompt(self, cursor):
""" Reimplemented for auto-indentation.
"""
super(FrontendWidget, self)._insert_continuation_prompt(cursor)
cursor.insertText(' ' * self._input_splitter.indent_spaces)
#---------------------------------------------------------------------------
# 'BaseFrontendMixin' abstract interface
#---------------------------------------------------------------------------
def _handle_complete_reply(self, rep):
""" Handle replies for tab completion.
"""
self.log.debug("complete: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('complete')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
text = '.'.join(self._get_context())
cursor.movePosition(QtGui.QTextCursor.Left, n=len(text))
self._complete_with_items(cursor, rep['content']['matches'])
def _silent_exec_callback(self, expr, callback):
"""Silently execute `expr` in the kernel and call `callback` with reply
the `expr` is evaluated silently in the kernel (without) output in
the frontend. Call `callback` with the
`repr <http://docs.python.org/library/functions.html#repr> `_ as first argument
Parameters
----------
expr : string
valid string to be executed by the kernel.
callback : function
function accepting one argument, as a string. The string will be
the `repr` of the result of evaluating `expr`
The `callback` is called with the `repr()` of the result of `expr` as
first argument. To get the object, do `eval()` on the passed value.
See Also
--------
_handle_exec_callback : private method, deal with calling callback with reply
"""
# generate uuid, which would be used as an indication of whether or
# not the unique request originated from here (can use msg id ?)
local_uuid = str(uuid.uuid1())
msg_id = self.kernel_client.execute(
'', silent=True, user_expressions={local_uuid: expr})
self._callback_dict[local_uuid] = callback
self._request_info['execute'][msg_id] = self._ExecutionRequest(
msg_id, 'silent_exec_callback')
def _handle_exec_callback(self, msg):
"""Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback``
Parameters
----------
msg : raw message send by the kernel containing an `user_expressions`
and having a 'silent_exec_callback' kind.
Notes
-----
This function will look for a `callback` associated with the
corresponding message id. Association has been made by
`_silent_exec_callback`. `callback` is then called with the `repr()`
of the value of corresponding `user_expressions` as argument.
`callback` is then removed from the known list so that any message
coming again with the same id won't trigger it.
"""
user_exp = msg['content'].get('user_expressions')
if not user_exp:
return
for expression in user_exp:
if expression in self._callback_dict:
self._callback_dict.pop(expression)(user_exp[expression])
def _handle_execute_reply(self, msg):
""" Handles replies for code execution.
"""
self.log.debug("execute: %s", msg.get('content', ''))
msg_id = msg['parent_header']['msg_id']
info = self._request_info['execute'].get(msg_id)
# unset reading flag, because if execute finished, raw_input can't
# still be pending.
self._reading = False
if info and info.kind == 'user' and not self._hidden:
# Make sure that all output from the SUB channel has been processed
# before writing a new prompt.
self.kernel_client.iopub_channel.flush()
# Reset the ANSI style information to prevent bad text in stdout
# from messing up our colors. We're not a true terminal so we're
# allowed to do this.
if self.ansi_codes:
self._ansi_processor.reset_sgr()
content = msg['content']
status = content['status']
if status == 'ok':
self._process_execute_ok(msg)
elif status == 'error':
self._process_execute_error(msg)
elif status == 'aborted':
self._process_execute_abort(msg)
self._show_interpreter_prompt_for_reply(msg)
self.executed.emit(msg)
self._request_info['execute'].pop(msg_id)
elif info and info.kind == 'silent_exec_callback' and not self._hidden:
self._handle_exec_callback(msg)
self._request_info['execute'].pop(msg_id)
else:
super(FrontendWidget, self)._handle_execute_reply(msg)
def _handle_input_request(self, msg):
""" Handle requests for raw_input.
"""
self.log.debug("input: %s", msg.get('content', ''))
if self._hidden:
raise RuntimeError(
'Request for raw input during hidden execution.')
# Make sure that all output from the SUB channel has been processed
# before entering readline mode.
self.kernel_client.iopub_channel.flush()
def callback(line):
self.kernel_client.stdin_channel.input(line)
if self._reading:
self.log.debug(
"Got second input request, assuming first was interrupted.")
self._reading = False
self._readline(msg['content']['prompt'], callback=callback)
def _kernel_restarted_message(self, died=True):
msg = "Kernel died, restarting" if died else "Kernel restarting"
self._append_html("<br>%s<hr><br>" % msg, before_prompt=False)
def _handle_kernel_died(self, since_last_heartbeat):
"""Handle the kernel's death (if we do not own the kernel).
"""
self.log.warn("kernel died: %s", since_last_heartbeat)
if self.custom_restart:
self.custom_restart_kernel_died.emit(since_last_heartbeat)
else:
self._kernel_restarted_message(died=True)
self.reset()
def _handle_kernel_restarted(self, died=True):
"""Notice that the autorestarter restarted the kernel.
There's nothing to do but show a message.
"""
self.log.warn("kernel restarted")
self._kernel_restarted_message(died=died)
self.reset()
def _handle_object_info_reply(self, rep):
""" Handle replies for call tips.
"""
self.log.debug("oinfo: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('call_tip')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
# Get the information for a call tip. For now we format the call
# line as string, later we can pass False to format_call and
# syntax-highlight it ourselves for nicer formatting in the
# calltip.
content = rep['content']
# if this is from pykernel, 'docstring' will be the only key
if content.get('ismagic', False):
# Don't generate a call-tip for magics. Ideally, we should
# generate a tooltip, but not on ( like we do for actual
# callables.
call_info, doc = None, None
else:
call_info, doc = call_tip(content, format_call=True)
if call_info or doc:
self._call_tip_widget.show_call_info(call_info, doc)
def _handle_pyout(self, msg):
""" Handle display hook output.
"""
self.log.debug("pyout: %s", msg.get('content', ''))
if not self._hidden and self._is_from_this_session(msg):
text = msg['content']['data']
self._append_plain_text(text + '\n', before_prompt=True)
def _handle_stream(self, msg):
""" Handle stdout, stderr, and stdin.
"""
self.log.debug("stream: %s", msg.get('content', ''))
if not self._hidden and self._is_from_this_session(msg):
# Most consoles treat tabs as being 8 space characters. Convert tabs
# to spaces so that output looks as expected regardless of this
# widget's tab width.
text = msg['content']['data'].expandtabs(8)
self._append_plain_text(text, before_prompt=True)
self._control.moveCursor(QtGui.QTextCursor.End)
def _handle_shutdown_reply(self, msg):
""" Handle shutdown signal, only if from other console.
"""
self.log.warn("shutdown: %s", msg.get('content', ''))
restart = msg.get('content', {}).get('restart', False)
if not self._hidden and not self._is_from_this_session(msg):
# got shutdown reply, request came from session other than ours
if restart:
# someone restarted the kernel, handle it
self._handle_kernel_restarted(died=False)
else:
# kernel was shutdown permanently
# this triggers exit_requested if the kernel was local,
# and a dialog if the kernel was remote,
# so we don't suddenly clear the qtconsole without asking.
if self._local_kernel:
self.exit_requested.emit(self)
else:
title = self.window().windowTitle()
reply = QtGui.QMessageBox.question(
self, title, "Kernel has been shutdown permanently. "
"Close the Console?", QtGui.QMessageBox.Yes,
QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.Yes:
self.exit_requested.emit(self)
def _handle_status(self, msg):
"""Handle status message"""
# This is where a busy/idle indicator would be triggered,
# when we make one.
state = msg['content'].get('execution_state', '')
if state == 'starting':
# kernel started while we were running
if self._executing:
self._handle_kernel_restarted(died=True)
elif state == 'idle':
pass
elif state == 'busy':
pass
def _started_channels(self):
""" Called when the KernelManager channels have started listening or
when the frontend is assigned an already listening KernelManager.
"""
self.reset(clear=True)
#---------------------------------------------------------------------------
# 'FrontendWidget' public interface
#---------------------------------------------------------------------------
def copy_raw(self):
""" Copy the currently selected text to the clipboard without attempting
to remove prompts or otherwise alter the text.
"""
self._control.copy()
def execute_file(self, path, hidden=False):
""" Attempts to execute file with 'path'. If 'hidden', no output is
shown.
"""
self.execute('execfile(%r)' % path, hidden=hidden)
def interrupt_kernel(self):
""" Attempts to interrupt the running kernel.
Also unsets _reading flag, to avoid runtime errors
if raw_input is called again.
"""
if self.custom_interrupt:
self._reading = False
self.custom_interrupt_requested.emit()
elif self.kernel_manager:
self._reading = False
self.kernel_manager.interrupt_kernel()
else:
self._append_plain_text(
'Cannot interrupt a kernel I did not start.\n')
def reset(self, clear=False):
""" Resets the widget to its initial state if ``clear`` parameter
is True, otherwise
prints a visual indication of the fact that the kernel restarted, but
does not clear the traces from previous usage of the kernel before it
was restarted. With ``clear=True``, it is similar to ``%clear``, but
also re-writes the banner and aborts execution if necessary.
"""
if self._executing:
self._executing = False
self._request_info['execute'] = {}
self._reading = False
self._highlighter.highlighting_on = False
if clear:
self._control.clear()
self._append_plain_text(self.banner)
# update output marker for stdout/stderr, so that startup
# messages appear after banner:
self._append_before_prompt_pos = self._get_cursor().position()
self._show_interpreter_prompt()
def restart_kernel(self, message, now=False):
""" Attempts to restart the running kernel.
"""
# FIXME: now should be configurable via a checkbox in the dialog. Right
# now at least the heartbeat path sets it to True and the manual restart
# to False. But those should just be the pre-selected states of a
# checkbox that the user could override if so desired. But I don't know
# enough Qt to go implementing the checkbox now.
if self.custom_restart:
self.custom_restart_requested.emit()
return
if self.kernel_manager:
# Pause the heart beat channel to prevent further warnings.
self.kernel_client.hb_channel.pause()
# Prompt the user to restart the kernel. Un-pause the heartbeat if
# they decline. (If they accept, the heartbeat will be un-paused
# automatically when the kernel is restarted.)
if self.confirm_restart:
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No
result = QtGui.QMessageBox.question(self, 'Restart kernel?',
message, buttons)
do_restart = result == QtGui.QMessageBox.Yes
else:
# confirm_restart is False, so we don't need to ask user
# anything, just do the restart
do_restart = True
if do_restart:
try:
self.kernel_manager.restart_kernel(now=now)
except RuntimeError as e:
self._append_plain_text('Error restarting kernel: %s\n' %
e,
before_prompt=True)
else:
self._append_html(
"<br>Restarting kernel...\n<hr><br>",
before_prompt=True,
)
else:
self.kernel_client.hb_channel.unpause()
else:
self._append_plain_text(
'Cannot restart a Kernel I did not start\n',
before_prompt=True)
#---------------------------------------------------------------------------
# 'FrontendWidget' protected interface
#---------------------------------------------------------------------------
def _call_tip(self):
""" Shows a call tip, if appropriate, at the current cursor location.
"""
# Decide if it makes sense to show a call tip
if not self.enable_calltips:
return False
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.Left)
if cursor.document().characterAt(cursor.position()) != '(':
return False
context = self._get_context(cursor)
if not context:
return False
# Send the metadata request to the kernel
name = '.'.join(context)
msg_id = self.kernel_client.object_info(name)
pos = self._get_cursor().position()
self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos)
return True
def _complete(self):
""" Performs completion at the current cursor location.
"""
context = self._get_context()
if context:
# Send the completion request to the kernel
msg_id = self.kernel_client.complete(
'.'.join(context), # text
self._get_input_buffer_cursor_line(), # line
self._get_input_buffer_cursor_column(), # cursor_pos
self.input_buffer) # block
pos = self._get_cursor().position()
info = self._CompletionRequest(msg_id, pos)
self._request_info['complete'] = info
def _get_context(self, cursor=None):
""" Gets the context for the specified cursor (or the current cursor
if none is specified).
"""
if cursor is None:
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.StartOfBlock,
QtGui.QTextCursor.KeepAnchor)
text = cursor.selection().toPlainText()
return self._completion_lexer.get_context(text)
def _process_execute_abort(self, msg):
""" Process a reply for an aborted execution request.
"""
self._append_plain_text("ERROR: execution aborted\n")
def _process_execute_error(self, msg):
""" Process a reply for an execution request that resulted in an error.
"""
content = msg['content']
# If a SystemExit is passed along, this means exit() was called - also
# all the ipython %exit magic syntax of '-k' to be used to keep
# the kernel running
if content['ename'] == 'SystemExit':
keepkernel = content['evalue'] == '-k' or content[
'evalue'] == 'True'
self._keep_kernel_on_exit = keepkernel
self.exit_requested.emit(self)
else:
traceback = ''.join(content['traceback'])
self._append_plain_text(traceback)
def _process_execute_ok(self, msg):
""" Process a reply for a successful execution request.
"""
payload = msg['content']['payload']
for item in payload:
if not self._process_execute_payload(item):
warning = 'Warning: received unknown payload of type %s'
print(warning % repr(item['source']))
def _process_execute_payload(self, item):
""" Process a single payload item from the list of payload items in an
execution reply. Returns whether the payload was handled.
"""
# The basic FrontendWidget doesn't handle payloads, as they are a
# mechanism for going beyond the standard Python interpreter model.
return False
def _show_interpreter_prompt(self):
""" Shows a prompt for the interpreter.
"""
self._show_prompt('>>> ')
def _show_interpreter_prompt_for_reply(self, msg):
""" Shows a prompt for the interpreter given an 'execute_reply' message.
"""
self._show_interpreter_prompt()
#------ Signal handlers ----------------------------------------------------
def _document_contents_change(self, position, removed, added):
""" Called whenever the document's content changes. Display a call tip
if appropriate.
"""
# Calculate where the cursor should be *after* the change:
position += added
document = self._control.document()
if position == self._get_cursor().position():
self._call_tip()
#------ Trait default initializers -----------------------------------------
def _banner_default(self):
""" Returns the standard Python banner.
"""
banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \
'"license" for more information.'
return banner % (sys.version, sys.platform)
def __init__(self, *args, **kwargs):
super(IPythonValidator, self).__init__(*args, **kwargs)
self.isp = IPythonInputSplitter()
from IPython.core.inputsplitter import IPythonInputSplitter
spliter = IPythonInputSplitter()
cell = "%time None"
print(spliter.transform_cell(cell))
# get_ipython().run_line_magic('time', 'None')
out = isp.transform_cell(raw)
self.assertEqual(out.rstrip(), expected.rstrip())
#-----------------------------------------------------------------------------
# Main - use as a script, mostly for developer experiments
#-----------------------------------------------------------------------------
if __name__ == '__main__':
# A simple demo for interactive experimentation. This code will not get
# picked up by any test suite.
from IPython.core.inputsplitter import IPythonInputSplitter
# configure here the syntax to use, prompt and whether to autoindent
#isp, start_prompt = InputSplitter(), '>>> '
isp, start_prompt = IPythonInputSplitter(), 'In> '
autoindent = True
#autoindent = False
try:
while True:
prompt = start_prompt
while isp.push_accepts_more():
indent = ' ' * isp.get_indent_spaces()
if autoindent:
line = indent + input(prompt + indent)
else:
line = input(prompt)
isp.push(line)
prompt = '... '
def execute_log(nb,
log,
secret='secret',
initial_env=None,
ignore_errors=False,
cwd=None,
test_dir=None,
variables=None):
"""
Executes a notebook from logged environments and returns the global environment that results from execution
Execute notebook & return the global environment that results from execution. If ``ignore_errors``
is ``True``, exceptions are swallowed. ``secret`` contains random digits so ``check_results`` and
``check`` are not easily modifiable. ``nb`` is passed in as a dictionary that's a parsed notebook
Args:
nb (``dict``): JSON representation of a notebook
log (``otter.check.logs.Log``): log from notebook execution
secret (``str``, optional): randomly generated integer used to rebind check function
initial_env (``str``, optional): name of initial environment
ignore_errors (``bool``, optional): whether exceptions should be ignored
cwd (``str``, optional): working directory of execution to be appended to ``sys.path`` in
grading environment
test_dir (``str``, optional): path to directory of tests in grading environment
variables (``dict``, optional): map of variable names -> type string to check type of deserialized
object to prevent arbitrary code from being put into the environment
Results:
``dict``: global environment resulting from executing all code of the input notebook
"""
with hide_outputs():
if initial_env:
global_env = initial_env.copy()
else:
global_env = {}
if test_dir:
source = f"import otter\ngrader = otter.Notebook(\"{test_dir}\")\n"
else:
source = f"import otter\ngrader = otter.Notebook()\n"
if cwd:
source += f"import sys\nsys.path.append(\"{cwd}\")\n"
logged_questions = []
m = mock.mock_open()
with mock.patch("otter.Notebook._log_event", m):
exec(source, global_env)
for cell in nb['cells']:
if cell['cell_type'] == 'code':
# transform the input to executable Python
# FIXME: use appropriate IPython functions here
isp = IPythonInputSplitter(line_input_checker=False)
code_lines = []
cell_source_lines = cell['source']
source_is_str_bool = False
if isinstance(cell_source_lines, str):
source_is_str_bool = True
cell_source_lines = cell_source_lines.split('\n')
# only execute import statements
cell_source_lines = [
re.sub(r"^\s+", "", l) for l in cell_source_lines
if "import" in l
]
for line in cell_source_lines:
try:
exec(line, global_env)
# source += cell_source
except:
if not ignore_errors:
raise
for entry in log.question_iterator():
shelf = entry.unshelve(global_env)
if variables is not None:
for k, v in shelf.items():
full_type = type(v).__module__ + "." + type(v).__name__
if not (k in variables and variables[k] == full_type):
del shelf[k]
print(
f"Found variable of different type than expected: {k}"
)
global_env.update(shelf)
global_env[f"check_results_{secret}"].append(
global_env["grader"].check(entry.question,
global_env=global_env))
logged_questions.append(entry.question)
print("Questions executed from log: {}".format(
", ".join(logged_questions)))
return global_env
def __init__(self, *args, **kwargs):
super(IPythonValidator, self).__init__(*args, **kwargs)
self.isp = IPythonInputSplitter()
def execute_notebook(nb,
secret='secret',
initial_env=None,
ignore_errors=False,
gradescope=False):
"""
Executes an ipython notebook and return the global environment that results from execution.
Execute notebook & return the global environment that results from execution.
TODO: write a note about the injection of check_results
If ignore_errors is True, exceptions are swallowed.
secret contains random digits so check_results and check are not easily modifiable
nb is passed in as a dictionary that's a parsed ipynb file
Args:
nb (dict of str: str): json representation of ipython notebook
secret (str, optional): randomly generated integer used to rebind check function
initial_env (str, optional): name of initial environment
ignore_errors (bool, optional): whether exceptions should be ignored
Results:
dict of str: object: global environment resulting from executing all code of the input notebook
"""
with hide_outputs():
if initial_env:
global_env = initial_env.copy()
else:
global_env = {}
source = ""
if gradescope:
source = "import sys\nsys.path.append(\"/autograder/submission\")\n"
# Before rewriting AST, find cells of code that generate errors.
# One round of execution is done beforehand to mimic the Jupyter notebook style of running
# (e.g. code runs up to the point of execution).
# The reason this is workaround is introduced is because once the
# source code is parsed into an AST, there is no sense of local cells
for cell in nb['cells']:
if cell['cell_type'] == 'code':
# transform the input to executable Python
# FIXME: use appropriate IPython functions here
isp = IPythonInputSplitter(line_input_checker=False)
try:
code_lines = []
cell_source_lines = cell['source']
source_is_str_bool = False
if isinstance(cell_source_lines, str):
source_is_str_bool = True
cell_source_lines = cell_source_lines.split('\n')
for line in cell_source_lines:
# Filter out ipython magic commands
# Filter out interact widget
if not line.startswith('%'):
if "interact(" not in line and not re.search(
r"otter\.Notebook\(.*?\)", line):
code_lines.append(line)
if source_is_str_bool:
code_lines.append('\n')
elif re.search(r"otter\.Notebook\(.*?\)", line):
if gradescope:
line = re.sub(
r"otter\.Notebook\(.*?\)",
"otter.Notebook(\"/autograder/submission/tests\")",
line)
else:
line = re.sub(
r"otter\.Notebook\(.*?\)",
"otter.Notebook(\"/home/tests\")",
line)
code_lines.append(line)
if source_is_str_bool:
code_lines.append('\n')
cell_source = isp.transform_cell(''.join(code_lines))
# patch otter.Notebook.export so that we don't create PDFs in notebooks
m = mock.mock_open()
with mock.patch('otter.Notebook.export', m):
exec(source + cell_source, global_env)
source += cell_source
except:
if not ignore_errors:
raise
tree = ast.parse(source)
# # CODE BELOW COMMENTED OUT BECAUSE the only check function is within the Notebook class
# if find_check_assignment(tree) or find_check_definition(tree):
# # an empty global_env will fail all the tests
# return global_env
# wrap check(..) calls into a check_results_X.append(check(..))
transformer = CheckCallWrapper(secret)
tree = transformer.visit(tree)
ast.fix_missing_locations(tree)
cleaned_source = compile(tree, filename="nb-ast", mode="exec")
try:
with open(os.devnull,
'w') as f, redirect_stdout(f), redirect_stderr(f):
# patch otter.Notebook.export so that we don't create PDFs in notebooks
m = mock.mock_open()
with mock.patch('otter.Notebook.export', m):
exec(cleaned_source, global_env)
except:
if not ignore_errors:
raise
return global_env
