def execute_notebook(nb_path): """Execute a notebook under grading conditions""" graded_nb_path = os.path.splitext(nb_path)[0] + '-graded.ipynb' nb_directory = os.path.split(nb_path)[0] # read in input notebook and check the source for shenanigans nb = nbformat.read(nb_path, as_version=4) source = "" for cell in nb.cells: if cell.cell_type != "code": continue isp = IPythonInputSplitter(line_input_checker=False) cell_source = isp.transform_cell(cell.source) source += cell_source tree = ast.parse(source) # no points for you if you try and cheat # XXX add a check for people importing a function called `check` if find_check_assignment(tree) or find_check_definition(tree): return # run the notebook with chdir(nb_directory): pm.execute_notebook(nb_path, graded_nb_path) graded_nb = nbformat.read(graded_nb_path, as_version=4) return graded_nb
def execute_notebook(nb, initial_env=None, ignore_errors=False): """ Execute notebook & return the global environment that results from execution. If ignore_errors is True, exceptions are swallowed. nb is passed in as a dictionary that's a parsed ipynb file """ with hide_outputs(): if initial_env: global_env = initial_env.copy() else: global_env = {} for cell in nb['cells']: if cell['cell_type'] == 'code': # transform the input to executable Python # FIXME: use appropriate IPython functions here isp = IPythonInputSplitter(line_input_checker=False) source = isp.transform_cell(''.join(cell['source'])) try: with open( '/dev/null', 'w') as f, redirect_stdout(f), redirect_stderr(f): exec(source, global_env) except: if not ignore_errors: raise return global_env
class PyDevFrontEnd: def __init__(self, *args, **kwargs): # Initialization based on: from IPython.testing.globalipapp import start_ipython self._curr_exec_line = 0 # Store certain global objects that IPython modifies _displayhook = sys.displayhook _excepthook = sys.excepthook # Create and initialize our IPython instance. shell = TerminalInteractiveShell.instance() # Create an intput splitter to handle input separation self.input_splitter = IPythonInputSplitter() shell.showtraceback = _showtraceback # IPython is ready, now clean up some global state... # Deactivate the various python system hooks added by ipython for # interactive convenience so we don't confuse the doctest system sys.displayhook = _displayhook sys.excepthook = _excepthook # So that ipython magics and aliases can be doctested (they work by making # a call into a global _ip object). Also make the top-level get_ipython # now return this without recursively calling here again. get_ipython = shell.get_ipython try: import __builtin__ except: import builtins as __builtin__ __builtin__._ip = shell __builtin__.get_ipython = get_ipython # We want to print to stdout/err as usual. io.stdout = original_stdout io.stderr = original_stderr self.ipython = shell def complete(self, string): return self.ipython.complete(None, line=string) def is_complete(self, string): return not self.input_splitter.push_accepts_more() def getNamespace(self): return self.ipython.user_ns def addExec(self, line): self.input_splitter.push(line) if self.is_complete(line): self.ipython.run_cell(self.input_splitter.source_reset()) return False else: return True
class PyDevFrontEnd: def __init__(self, *args, **kwargs): # Initialization based on: from IPython.testing.globalipapp import start_ipython self._curr_exec_line = 0 # Store certain global objects that IPython modifies _displayhook = sys.displayhook _excepthook = sys.excepthook # Create and initialize our IPython instance. shell = TerminalInteractiveShell.instance() # Create an intput splitter to handle input separation self.input_splitter = IPythonInputSplitter() shell.showtraceback = _showtraceback # IPython is ready, now clean up some global state... # Deactivate the various python system hooks added by ipython for # interactive convenience so we don't confuse the doctest system sys.displayhook = _displayhook sys.excepthook = _excepthook # So that ipython magics and aliases can be doctested (they work by making # a call into a global _ip object). Also make the top-level get_ipython # now return this without recursively calling here again. get_ipython = shell.get_ipython try: import __builtin__ except: import builtins as __builtin__ __builtin__._ip = shell __builtin__.get_ipython = get_ipython # We want to print to stdout/err as usual. io.stdout = original_stdout io.stderr = original_stderr self.ipython = shell def complete(self, string): return self.ipython.complete(None, line=string) def is_complete(self, string): return not self.input_splitter.push_accepts_more() def getNamespace(self): return self.ipython.user_ns def addExec(self, line): self.input_splitter.push(line) if self.is_complete(line): self.ipython.run_cell(self.input_splitter.source_reset()) return False else: return True
def execute(code, namespace): isp = IPythonInputSplitter() kernel_manager = MyInProcessKernelManager() kernel_manager.start_kernel(namespace) kernel = kernel_manager.kernel def do_execute(): result = kernel.shell.run_cell(cell, False) if result.error_before_exec is not None: err = result.error_before_exec else: err = result.error_in_exec if not result.success: if kernel.shell._last_traceback: for tb in kernel.shell._last_traceback: print(tb) #TODO: log for line in code.splitlines(): if isp.push_accepts_more(): isp.push(line.strip("\n")) continue cell = isp.source_reset() do_execute() isp.push(line.strip("\n")) cell = isp.source_reset() if len(cell): do_execute() return namespace
def _ipython_to_python(source): try: from IPython.core.inputsplitter import IPythonInputSplitter except ImportError: log.warning("IPython is required to process Notebook source code - " "install it by running 'pip install ipython'") raise MissingIPython() else: isp = IPythonInputSplitter(line_input_checker=False) python_source = isp.transform_cell(source) return _fix_ipython_to_python_lf(python_source, source)
def ipython2encodedpython(code): def tweak_transform(orig_transform): """ Takes the transform and modifies it such that we compare each line to its transformation. If they are different, that means the line is a special ipython command. We strip that from the output, but record the special command in a comment so we can restore it. """ def new_push_builder(push_func): def new_push(line): result = push_func(line) if line != result: return "# EPY: ESCAPE {}".format(line) return result return new_push orig_transform.push = functools.update_wrapper( new_push_builder(orig_transform.push), orig_transform.push) return orig_transform from IPython.core.inputtransformer import StatelessInputTransformer @StatelessInputTransformer.wrap def escaped_epy_lines(line): """Transform lines that happen to look like EPY comments.""" if line.startswith("# EPY"): return "# EPY: ESCAPE {}".format(line) return line """Transform IPython syntax to an encoded Python syntax Parameters ---------- code : str IPython code, to be transformed to Python encoded in a way to facilitate transformation back into IPython. """ from IPython.core.inputsplitter import IPythonInputSplitter # get a list of default line transforms. then capture fake_isp = IPythonInputSplitter(line_input_checker=False) logical_line_transforms = [escaped_epy_lines()] logical_line_transforms.extend([ tweak_transform(transform) for transform in fake_isp.logical_line_transforms ]) isp = IPythonInputSplitter(line_input_checker=False, logical_line_transforms=logical_line_transforms) result = isp.transform_cell(code) return result
def _is_pdb_complete(self, source): """ Check if the pdb input is ready to be executed. """ if source and source[0] == '!': source = source[1:] if PY2: tm = IPythonInputSplitter() else: tm = TransformerManager() complete, indent = tm.check_complete(source) if indent is not None: indent = indent * ' ' return complete != 'incomplete', indent
def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False): """ Execute notebook & return the global environment that results from execution. TODO: write a note about the injection of check_results If ignore_errors is True, exceptions are swallowed. secret contains random digits so check_results and check are not easily modifiable nb is passed in as a dictionary that's a parsed ipynb file """ with hide_outputs(): if initial_env: global_env = initial_env.copy() else: global_env = {} source = "" # Before rewriting AST, find cells of code that generate errors. # One round of execution is done beforehand to mimic the Jupyter notebook style of running # (e.g. code runs up to the point of execution). # The reason this is workaround is introduced is because once the # source code is parsed into an AST, there is no sense of local cells exec("from gofer.ok import check", global_env) exec("from IPython.display import display", global_env) for cell in nb['cells']: if cell['cell_type'] == 'code': # transform the input to executable Python # FIXME: use appropriate IPython functions here isp = IPythonInputSplitter(line_input_checker=False) try: code_lines = [] for line in cell['source']: # Filter out ipython magic commands if not line.startswith('%'): code_lines.append(ok_grade_to_check(line)) cell_source = isp.transform_cell(''.join(code_lines)) # exec(cell_source, global_env) run_this_block(cell_source, secret, global_env, ignore_errors) source += cell_source except Exception as e: logging.error(e) traceback.print_exc() if not ignore_errors: raise return global_env
class IPythonValidator(PythonValidator): def __init__(self, *args, **kwargs): super(IPythonValidator, self).__init__(*args, **kwargs) self.isp = IPythonInputSplitter() def validate(self, document): document = Document(text=self.isp.transform_cell(document.text)) super(IPythonValidator, self).validate(document)
def __init__(self, pydev_host, pydev_client_port, *args, **kwarg): # Create and initialize our IPython instance. self.ipython = PyDevTerminalInteractiveShell.instance() # Back channel to PyDev to open editors (in the future other # info may go back this way. This is the same channel that is # used to get stdin, see StdIn in pydev_console_utils) self.ipython.set_hook( 'editor', create_editor_hook(pydev_host, pydev_client_port)) # Create an input splitter to handle input separation self.input_splitter = IPythonInputSplitter() # Display the IPython banner, this has version info and # help info self.ipython.show_banner()
class IPythonValidator(PythonValidator): def __init__(self, *args, **kwargs): super(IPythonValidator, self).__init__(*args, **kwargs) self.isp = IPythonInputSplitter() def validate(self, document): document = Document(text=self.isp.transform_cell(document.text)) super(IPythonValidator, self).validate(document)
def notebook_path_to_dependencies(path_to_notebook, remap=True, custom_namespaces=None): """Helper function that turns a jupyter notebook into a list of dependencies Parameters ---------- path_to_notebook : str remap : bool, optional Normalize the import names to be synonymous with their conda/pip names custom_namespaces : list of str ot None, optional If not None, then resulting package outputs will list everying under these namespaces (e.g., for packages foo.bar and foo.baz, the outputs are foo.bar and foo.baz instead of foo if custom_namespaces=["foo"]). Returns ------- dict Dict of dependencies keyed on - 'builtin' - libraries built in to python - 'required' - libraries that are found at the top level of your modules - 'questionable' - libraries that are found inside try/except blocks - 'relative' - libraries that are relative imports Examples -------- >>> depfinder.notebook_path_to_dependencies('depfinder_usage.ipynb') {'builtin': ['os', 'pprint'], 'required': ['depfinder']} """ try: from IPython.core.inputsplitter import IPythonInputSplitter transform = IPythonInputSplitter( line_input_checker=False).transform_cell except: transform = lambda code: code nb = json.load(io.open(path_to_notebook, encoding='utf8')) codeblocks = [ ''.join(cell['source']) for cell in nb['cells'] if cell['cell_type'] == 'code' ] all_deps = defaultdict(set) for codeblock in codeblocks: codeblock = transform(codeblock) # TODO this may fail on py2/py3 syntax when running in the other runtime. # May want to consider updating some error handling around that case. # Will wait until that use case surfaces before modifying deps_dict = get_imported_libs( codeblock, custom_namespaces=custom_namespaces).describe() for k, v in deps_dict.items(): all_deps[k].update(v) all_deps = {k: sorted(list(v)) for k, v in all_deps.items()} if remap: return sanitize_deps(all_deps) return all_deps
def ipython2python(code): """Transform IPython syntax to pure Python syntax Parameters ---------- code : str IPython code, to be transformed to pure Python """ try: from IPython.core.inputsplitter import IPythonInputSplitter except ImportError: warnings.warn( "IPython is needed to transform IPython syntax to pure Python." " Install ipython if you need this functionality.") return code else: isp = IPythonInputSplitter(line_input_checker=False) return isp.transform_cell(code)
def ipython2python(code): """Transform IPython syntax to pure Python syntax Parameters ---------- code : str IPython code, to be transformed to pure Python """ try: from IPython.core.inputsplitter import IPythonInputSplitter except ImportError: warnings.warn( "IPython is needed to transform IPython syntax to pure Python." " Install ipython if you need this functionality." ) return code else: isp = IPythonInputSplitter(line_input_checker=False) return isp.transform_cell(code)
def transform_cell(code, indent_only=False): """Transform ipython code to python code.""" number_empty_lines = count_leading_empty_lines(code) if indent_only: # Not implemented for PY2 if LooseVersion(ipy_version) < LooseVersion('7.0.0'): return code if not code.endswith('\n'): code += '\n' # Ensure the cell has a trailing newline lines = code.splitlines(keepends=True) lines = leading_indent(leading_empty_lines(lines)) code = ''.join(lines) else: if LooseVersion(ipy_version) < LooseVersion('7.0.0'): tm = IPythonInputSplitter() return tm.transform_cell(code) else: tm = TransformerManager() code = tm.transform_cell(code) return '\n' * number_empty_lines + code
def execute(code): isp = IPythonInputSplitter() def do_execute(): #print("CELL", cell ) result = kernel.shell.run_cell(cell, False) if result.error_before_exec is not None: err = result.error_before_exec else: err = result.error_in_exec if not result.success: for tb in kernel.shell._last_traceback: print(tb) return False return True for line in code.splitlines(): if isp.push_accepts_more(): isp.push(line.strip("\n")) continue cell = isp.source_reset() if not do_execute(): return False cell = isp.source_reset() if len(cell): return do_execute() else: return True
def ipython2python(code): isp = IPythonInputSplitter() newcode = "" for line in code.splitlines(): if isp.push_accepts_more(): isp.push(line.strip("\n")) continue cell = isp.source_reset() if cell.startswith("get_ipython().run_"): cell = "_ = " + cell newcode += cell + "\n" isp.push(line.strip("\n")) cell = isp.source_reset() if len(cell): if cell.startswith("get_ipython().run_"): cell = "_ = " + cell newcode += cell return newcode
def __init__(self, *args, **kwargs): # Initialization based on: from IPython.testing.globalipapp import start_ipython self._curr_exec_line = 0 # Store certain global objects that IPython modifies _displayhook = sys.displayhook _excepthook = sys.excepthook # Create and initialize our IPython instance. shell = TerminalInteractiveShell.instance() # Create an intput splitter to handle input separation self.input_splitter = IPythonInputSplitter() shell.showtraceback = _showtraceback # IPython is ready, now clean up some global state... # Deactivate the various python system hooks added by ipython for # interactive convenience so we don't confuse the doctest system sys.displayhook = _displayhook sys.excepthook = _excepthook # So that ipython magics and aliases can be doctested (they work by making # a call into a global _ip object). Also make the top-level get_ipython # now return this without recursively calling here again. get_ipython = shell.get_ipython try: import __builtin__ except: import builtins as __builtin__ __builtin__._ip = shell __builtin__.get_ipython = get_ipython # We want to print to stdout/err as usual. io.stdout = original_stdout io.stderr = original_stderr self.ipython = shell
def __init__(self, pydev_host, pydev_client_port, *args, **kwarg): # Create and initialize our IPython instance. self.ipython = PyDevTerminalInteractiveShell.instance() # Back channel to PyDev to open editors (in the future other # info may go back this way. This is the same channel that is # used to get stdin, see StdIn in pydev_console_utils) self.ipython.set_hook('editor', create_editor_hook(pydev_host, pydev_client_port)) # Create an input splitter to handle input separation self.input_splitter = IPythonInputSplitter() # Display the IPython banner, this has version info and # help info self.ipython.show_banner()
def notebook_path_to_dependencies(path_to_notebook, remap=True): """Helper function that turns a jupyter notebook into a list of dependencies Parameters ---------- path_to_notebook : str remap : bool, optional Normalize the import names to be synonymous with their conda/pip names Returns ------- dict Dict of dependencies keyed on - 'builtin' - libraries built in to python - 'required' - libraries that are found at the top level of your modules - 'questionable' - libraries that are found inside try/except blocks - 'relative' - libraries that are relative imports Examples -------- >>> depfinder.notebook_path_to_dependencies('depfinder_usage.ipynb') {'builtin': ['os', 'pprint'], 'required': ['depfinder']} """ try: from IPython.core.inputsplitter import IPythonInputSplitter transform = IPythonInputSplitter(line_input_checker=False).transform_cell except: transform = lambda code: code nb = json.load(io.open(path_to_notebook, encoding='utf8')) codeblocks = [''.join(cell['source']) for cell in nb['cells'] if cell['cell_type'] == 'code'] all_deps = defaultdict(set) for codeblock in codeblocks: codeblock = transform(codeblock) deps_dict = get_imported_libs(codeblock).describe() for k, v in deps_dict.items(): all_deps[k].update(v) all_deps = {k: sorted(list(v)) for k, v in all_deps.items()} if remap: return sanitize_deps(all_deps) return all_deps
def __init__(self, pydev_host, pydev_client_port, exec_queue, *args, **kwarg): self.exec_queue = exec_queue self._curr_exec_line = 0 # Store certain global objects that IPython modifies _displayhook = sys.displayhook _excepthook = sys.excepthook # Create and initialize our IPython instance. self.ipython = PyDevTerminalInteractiveShell.instance() # Back channel to PyDev to open editors (in the future other # info may go back this way. This is the same channel that is # used to get stdin, see StdIn in pydev_console_utils) self.ipython.set_hook('editor', create_editor_hook(pydev_host, pydev_client_port)) # Create an input splitter to handle input separation self.input_splitter = IPythonInputSplitter() # Display the IPython banner, this has version info and # help info self.ipython.show_banner() # IPython is ready, now clean up some global state... # Deactivate the various python system hooks added by ipython for # interactive convenience so we don't confuse the doctest system sys.displayhook = _displayhook sys.excepthook = _excepthook # So that ipython magics and aliases can be doctested (they work by making # a call into a global _ip object). Also make the top-level get_ipython # now return this without recursively calling here again. try: import __builtin__ except: import builtins as __builtin__ __builtin__._ip = self.ipython __builtin__.get_ipython = self.ipython.get_ipython # We want to print to stdout/err as usual. io.stdout = original_stdout io.stderr = original_stderr
def __init__(self, *args, **kwargs): # Initialization based on: from IPython.testing.globalipapp import start_ipython self._curr_exec_line = 0 # Store certain global objects that IPython modifies _displayhook = sys.displayhook _excepthook = sys.excepthook # Create and initialize our IPython instance. shell = TerminalInteractiveShell.instance() # Create an intput splitter to handle input separation self.input_splitter = IPythonInputSplitter() shell.showtraceback = _showtraceback # IPython is ready, now clean up some global state... # Deactivate the various python system hooks added by ipython for # interactive convenience so we don't confuse the doctest system sys.displayhook = _displayhook sys.excepthook = _excepthook # So that ipython magics and aliases can be doctested (they work by making # a call into a global _ip object). Also make the top-level get_ipython # now return this without recursively calling here again. get_ipython = shell.get_ipython try: import __builtin__ except: import builtins as __builtin__ __builtin__._ip = shell __builtin__.get_ipython = get_ipython # We want to print to stdout/err as usual. io.stdout = original_stdout io.stderr = original_stderr self.ipython = shell
class PyDevFrontEnd: def __init__(self, pydev_host, pydev_client_port, *args, **kwarg): # Create and initialize our IPython instance. self.ipython = PyDevTerminalInteractiveShell.instance() # Back channel to PyDev to open editors (in the future other # info may go back this way. This is the same channel that is # used to get stdin, see StdIn in pydev_console_utils) self.ipython.set_hook( 'editor', create_editor_hook(pydev_host, pydev_client_port)) # Create an input splitter to handle input separation self.input_splitter = IPythonInputSplitter() # Display the IPython banner, this has version info and # help info self.ipython.show_banner() def complete(self, string): return self.ipython.complete(None, line=string) def getCompletions(self, text, act_tok): # Get completions from IPython and from PyDev and merge the results # IPython only gives context free list of completions, while PyDev # gives detailed information about completions. try: TYPE_IPYTHON = '11' TYPE_IPYTHON_MAGIC = '12' _line, ipython_completions = self.complete(text) from _pydev_completer import Completer completer = Completer(self.getNamespace(), None) ret = completer.complete(act_tok) append = ret.append ip = self.ipython pydev_completions = set([f[0] for f in ret]) for ipython_completion in ipython_completions: if ipython_completion not in pydev_completions: pydev_completions.add(ipython_completion) inf = ip.object_inspect(ipython_completion) if inf['type_name'] == 'Magic function': pydev_type = TYPE_IPYTHON_MAGIC else: pydev_type = TYPE_IPYTHON pydev_doc = inf['docstring'] if pydev_doc is None: pydev_doc = '' append((ipython_completion, pydev_doc, '', pydev_type)) return ret except: import traceback traceback.print_exc() return [] def getNamespace(self): return self.ipython.user_ns def addExec(self, line): self.input_splitter.push(line) if not self.input_splitter.push_accepts_more(): self.ipython.run_cell(self.input_splitter.source_reset(), store_history=True) return False else: return True
def module_from_spec(spec): return _SpecMethods(spec).create() def _init_module_attrs(spec, module): return _SpecMethods(spec).init_module_attrs(module) try: import IPython from IPython.core.inputsplitter import IPythonInputSplitter dedent = IPythonInputSplitter( line_input_checker=False, physical_line_transforms=[ IPython.core.inputsplitter.leading_indent(), IPython.core.inputsplitter.ipy_prompt(), IPython.core.inputsplitter.cellmagic(end_on_blank_line=False), ], ).transform_cell except: from textwrap import dedent __all__ = "Notebook", "reload" class FinderContextManager: """ FinderContextManager is the base class for the notebook loader. It provides a context manager that replaces `FileFinder` in the `sys.path_hooks` to include an instance of the class in the python findering system.
def __init__(self, shell): super(TerminalMagics, self).__init__(shell) self.input_splitter = IPythonInputSplitter()
def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False, cwd=None, test_dir=None, seed=None): """ Executes a notebook and returns the global environment that results from execution Execute notebook & return the global environment that results from execution. If ``ignore_errors`` is ``True``, exceptions are swallowed. ``secret`` contains random digits so ``check_results`` and ``check`` are not easily modifiable. ``nb`` is passed in as a dictionary that's a parsed notebook Args: nb (``dict``): JSON representation of a notebook secret (``str``, optional): randomly generated integer used to rebind check function initial_env (``str``, optional): name of initial environment ignore_errors (``bool``, optional): whether exceptions should be ignored cwd (``str``, optional): working directory of execution to be appended to ``sys.path`` in grading environment test_dir (``str``, optional): path to directory of tests in grading environment seed (``int``, optional): random seed for intercell seeding Results: ``dict``: global environment resulting from executing all code of the input notebook """ with hide_outputs(): if initial_env: global_env = initial_env.copy() else: global_env = {} # add display from IPython global_env["display"] = display source = "" # if gradescope: # source = "import sys\nsys.path.append(\"/autograder/submission\")\n" # el if cwd: source = f"import sys\nsys.path.append(r\"{cwd}\")\n" exec(source, global_env) if seed is not None: # source += "import numpy as np\nimport random\n" import numpy as np import random global_env["np"] = np global_env["random"] = random # Before rewriting AST, find cells of code that generate errors. # One round of execution is done beforehand to mimic the Jupyter notebook style of running # (e.g. code runs up to the point of execution). # The reason this is workaround is introduced is because once the # source code is parsed into an AST, there is no sense of local cells for cell in nb['cells']: if cell['cell_type'] == 'code': # transform the input to executable Python # FIXME: use appropriate IPython functions here isp = IPythonInputSplitter(line_input_checker=False) try: code_lines = [] cell_source_lines = cell['source'] source_is_str_bool = False if isinstance(cell_source_lines, str): source_is_str_bool = True cell_source_lines = cell_source_lines.split('\n') for line in cell_source_lines: # Filter out ipython magic commands # Filter out interact widget if not line.startswith('%'): if "interact(" not in line and not re.search(r"otter\.Notebook\(.*?\)", line): code_lines.append(line) if source_is_str_bool: code_lines.append('\n') elif re.search(r"otter\.Notebook\(.*?\)", line): # TODO: move this check into CheckCallWrapper # if gradescope: # line = re.sub(r"otter\.Notebook\(.*?\)", "otter.Notebook(\"/autograder/submission/tests\")", line) # el if test_dir: line = re.sub(r"otter\.Notebook\(.*?\)", f"otter.Notebook(\"{test_dir}\")", line) else: line = re.sub(r"otter\.Notebook\(.*?\)", "otter.Notebook(\"/home/tests\")", line) code_lines.append(line) if source_is_str_bool: code_lines.append('\n') if seed is not None: cell_source = "np.random.seed({})\nrandom.seed({})\n".format(seed, seed) + isp.transform_cell(''.join(code_lines)) else: cell_source = isp.transform_cell(''.join(code_lines)) # patch otter.Notebook.export so that we don't create PDFs in notebooks # TODO: move this patch into CheckCallWrapper m = mock.mock_open() with mock.patch('otter.Notebook.export', m), mock.patch("otter.Notebook._log_event", m): exec(cell_source, global_env) source += cell_source except: if not ignore_errors: raise tree = ast.parse(source) # # CODE BELOW COMMENTED OUT BECAUSE the only check function is within the Notebook class # if find_check_assignment(tree) or find_check_definition(tree): # # an empty global_env will fail all the tests # return global_env # wrap check(..) calls into a check_results_X.append(check(..)) transformer = CheckCallWrapper(secret) tree = transformer.visit(tree) ast.fix_missing_locations(tree) cleaned_source = compile(tree, filename="nb-ast", mode="exec") try: with open(os.devnull, 'w') as f, redirect_stdout(f), redirect_stderr(f): # patch otter.Notebook.export so that we don't create PDFs in notebooks m = mock.mock_open() with mock.patch('otter.Notebook.export', m), mock.patch("otter.Notebook._log_event", m): exec(cleaned_source, global_env) except: if not ignore_errors: raise return global_env
class IPythonWidget(FrontendWidget): """ A FrontendWidget for an IPython kernel. """ # If set, the 'custom_edit_requested(str, int)' signal will be emitted when # an editor is needed for a file. This overrides 'editor' and 'editor_line' # settings. custom_edit = Bool(False) custom_edit_requested = QtCore.Signal(object, object) editor = Unicode(default_editor, config=True, help=""" A command for invoking a system text editor. If the string contains a {filename} format specifier, it will be used. Otherwise, the filename will be appended to the end the command. """) editor_line = Unicode(config=True, help=""" The editor command to use when a specific line number is requested. The string should contain two format specifiers: {line} and {filename}. If this parameter is not specified, the line number option to the %edit magic will be ignored. """) style_sheet = Unicode(config=True, help=""" A CSS stylesheet. The stylesheet can contain classes for: 1. Qt: QPlainTextEdit, QFrame, QWidget, etc 2. Pygments: .c, .k, .o, etc. (see PygmentsHighlighter) 3. IPython: .error, .in-prompt, .out-prompt, etc """) syntax_style = Unicode(config=True, help=""" If not empty, use this Pygments style for syntax highlighting. Otherwise, the style sheet is queried for Pygments style information. """) # Prompts. in_prompt = Unicode(default_in_prompt, config=True) out_prompt = Unicode(default_out_prompt, config=True) input_sep = Unicode(default_input_sep, config=True) output_sep = Unicode(default_output_sep, config=True) output_sep2 = Unicode(default_output_sep2, config=True) # FrontendWidget protected class variables. _input_splitter_class = IPythonInputSplitter _prompt_transformer = IPythonInputSplitter( physical_line_transforms=[ipy_prompt()], logical_line_transforms=[], python_line_transforms=[], ) # IPythonWidget protected class variables. _PromptBlock = namedtuple('_PromptBlock', ['block', 'length', 'number']) _payload_source_edit = 'edit_magic' _payload_source_exit = 'ask_exit' _payload_source_next_input = 'set_next_input' _payload_source_page = 'page' _retrying_history_request = False #--------------------------------------------------------------------------- # 'object' interface #--------------------------------------------------------------------------- def __init__(self, *args, **kw): super(IPythonWidget, self).__init__(*args, **kw) # IPythonWidget protected variables. self._payload_handlers = { self._payload_source_edit: self._handle_payload_edit, self._payload_source_exit: self._handle_payload_exit, self._payload_source_page: self._handle_payload_page, self._payload_source_next_input: self._handle_payload_next_input } self._previous_prompt_obj = None self._keep_kernel_on_exit = None # Initialize widget styling. if self.style_sheet: self._style_sheet_changed() self._syntax_style_changed() else: self.set_default_style() #--------------------------------------------------------------------------- # 'BaseFrontendMixin' abstract interface #--------------------------------------------------------------------------- def _handle_complete_reply(self, rep): """ Reimplemented to support IPython's improved completion machinery. """ self.log.debug("complete: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('complete') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): matches = rep['content']['matches'] text = rep['content']['matched_text'] offset = len(text) # Clean up matches with period and path separators if the matched # text has not been transformed. This is done by truncating all # but the last component and then suitably decreasing the offset # between the current cursor position and the start of completion. if len(matches) > 1 and matches[0][:offset] == text: parts = re.split(r'[./\\]', text) sep_count = len(parts) - 1 if sep_count: chop_length = sum(map(len, parts[:sep_count])) + sep_count matches = [match[chop_length:] for match in matches] offset -= chop_length # Move the cursor to the start of the match and complete. cursor.movePosition(QtGui.QTextCursor.Left, n=offset) self._complete_with_items(cursor, matches) def _handle_execute_reply(self, msg): """ Reimplemented to support prompt requests. """ msg_id = msg['parent_header'].get('msg_id') info = self._request_info['execute'].get(msg_id) if info and info.kind == 'prompt': number = msg['content']['execution_count'] + 1 self._show_interpreter_prompt(number) self._request_info['execute'].pop(msg_id) else: super(IPythonWidget, self)._handle_execute_reply(msg) def _handle_history_reply(self, msg): """ Implemented to handle history tail replies, which are only supported by the IPython kernel. """ content = msg['content'] if 'history' not in content: self.log.error("History request failed: %r" % content) if content.get('status', '') == 'aborted' and \ not self._retrying_history_request: # a *different* action caused this request to be aborted, so # we should try again. self.log.error("Retrying aborted history request") # prevent multiple retries of aborted requests: self._retrying_history_request = True # wait out the kernel's queue flush, which is currently timed at 0.1s time.sleep(0.25) self.kernel_client.shell_channel.history( hist_access_type='tail', n=1000) else: self._retrying_history_request = False return # reset retry flag self._retrying_history_request = False history_items = content['history'] self.log.debug("Received history reply with %i entries", len(history_items)) items = [] last_cell = u"" for _, _, cell in history_items: cell = cell.rstrip() if cell != last_cell: items.append(cell) last_cell = cell self._set_history(items) def _handle_pyout(self, msg): """ Reimplemented for IPython-style "display hook". """ self.log.debug("pyout: %s", msg.get('content', '')) if not self._hidden and self._is_from_this_session(msg): content = msg['content'] prompt_number = content.get('execution_count', 0) data = content['data'] if 'text/html' in data: self._append_plain_text(self.output_sep, True) self._append_html(self._make_out_prompt(prompt_number), True) html = data['text/html'] self._append_plain_text('\n', True) self._append_html(html + self.output_sep2, True) elif 'text/plain' in data: self._append_plain_text(self.output_sep, True) self._append_html(self._make_out_prompt(prompt_number), True) text = data['text/plain'] # If the repr is multiline, make sure we start on a new line, # so that its lines are aligned. if "\n" in text and not self.output_sep.endswith("\n"): self._append_plain_text('\n', True) self._append_plain_text(text + self.output_sep2, True) def _handle_display_data(self, msg): """ The base handler for the ``display_data`` message. """ self.log.debug("display: %s", msg.get('content', '')) # For now, we don't display data from other frontends, but we # eventually will as this allows all frontends to monitor the display # data. But we need to figure out how to handle this in the GUI. if not self._hidden and self._is_from_this_session(msg): source = msg['content']['source'] data = msg['content']['data'] metadata = msg['content']['metadata'] # In the regular IPythonWidget, we simply print the plain text # representation. if 'text/html' in data: html = data['text/html'] self._append_html(html, True) elif 'text/plain' in data: text = data['text/plain'] self._append_plain_text(text, True) # This newline seems to be needed for text and html output. self._append_plain_text(u'\n', True) def _started_channels(self): """Reimplemented to make a history request and load %guiref.""" super(IPythonWidget, self)._started_channels() self._load_guiref_magic() self.kernel_client.shell_channel.history(hist_access_type='tail', n=1000) def _started_kernel(self): """Load %guiref when the kernel starts (if channels are also started). Principally triggered by kernel restart. """ if self.kernel_client.shell_channel is not None: self._load_guiref_magic() def _load_guiref_magic(self): """Load %guiref magic.""" self.kernel_client.shell_channel.execute('\n'.join([ "try:", " _usage", "except:", " from IPython.core import usage as _usage", " get_ipython().register_magic_function(_usage.page_guiref, 'line', 'guiref')", " del _usage", ]), silent=True) #--------------------------------------------------------------------------- # 'ConsoleWidget' public interface #--------------------------------------------------------------------------- #--------------------------------------------------------------------------- # 'FrontendWidget' public interface #--------------------------------------------------------------------------- def execute_file(self, path, hidden=False): """ Reimplemented to use the 'run' magic. """ # Use forward slashes on Windows to avoid escaping each separator. if sys.platform == 'win32': path = os.path.normpath(path).replace('\\', '/') # Perhaps we should not be using %run directly, but while we # are, it is necessary to quote or escape filenames containing spaces # or quotes. # In earlier code here, to minimize escaping, we sometimes quoted the # filename with single quotes. But to do this, this code must be # platform-aware, because run uses shlex rather than python string # parsing, so that: # * In Win: single quotes can be used in the filename without quoting, # and we cannot use single quotes to quote the filename. # * In *nix: we can escape double quotes in a double quoted filename, # but can't escape single quotes in a single quoted filename. # So to keep this code non-platform-specific and simple, we now only # use double quotes to quote filenames, and escape when needed: if ' ' in path or "'" in path or '"' in path: path = '"%s"' % path.replace('"', '\\"') self.execute('%%run %s' % path, hidden=hidden) #--------------------------------------------------------------------------- # 'FrontendWidget' protected interface #--------------------------------------------------------------------------- def _complete(self): """ Reimplemented to support IPython's improved completion machinery. """ # We let the kernel split the input line, so we *always* send an empty # text field. Readline-based frontends do get a real text field which # they can use. text = '' # Send the completion request to the kernel msg_id = self.kernel_client.shell_channel.complete( text, # text self._get_input_buffer_cursor_line(), # line self._get_input_buffer_cursor_column(), # cursor_pos self.input_buffer) # block pos = self._get_cursor().position() info = self._CompletionRequest(msg_id, pos) self._request_info['complete'] = info def _process_execute_error(self, msg): """ Reimplemented for IPython-style traceback formatting. """ content = msg['content'] traceback = '\n'.join(content['traceback']) + '\n' if False: # FIXME: For now, tracebacks come as plain text, so we can't use # the html renderer yet. Once we refactor ultratb to produce # properly styled tracebacks, this branch should be the default traceback = traceback.replace(' ', ' ') traceback = traceback.replace('\n', '<br/>') ename = content['ename'] ename_styled = '<span class="error">%s</span>' % ename traceback = traceback.replace(ename, ename_styled) self._append_html(traceback) else: # This is the fallback for now, using plain text with ansi escapes self._append_plain_text(traceback) def _process_execute_payload(self, item): """ Reimplemented to dispatch payloads to handler methods. """ handler = self._payload_handlers.get(item['source']) if handler is None: # We have no handler for this type of payload, simply ignore it return False else: handler(item) return True def _show_interpreter_prompt(self, number=None): """ Reimplemented for IPython-style prompts. """ # If a number was not specified, make a prompt number request. if number is None: msg_id = self.kernel_client.shell_channel.execute('', silent=True) info = self._ExecutionRequest(msg_id, 'prompt') self._request_info['execute'][msg_id] = info return # Show a new prompt and save information about it so that it can be # updated later if the prompt number turns out to be wrong. self._prompt_sep = self.input_sep self._show_prompt(self._make_in_prompt(number), html=True) block = self._control.document().lastBlock() length = len(self._prompt) self._previous_prompt_obj = self._PromptBlock(block, length, number) # Update continuation prompt to reflect (possibly) new prompt length. self._set_continuation_prompt(self._make_continuation_prompt( self._prompt), html=True) def _show_interpreter_prompt_for_reply(self, msg): """ Reimplemented for IPython-style prompts. """ # Update the old prompt number if necessary. content = msg['content'] # abort replies do not have any keys: if content['status'] == 'aborted': if self._previous_prompt_obj: previous_prompt_number = self._previous_prompt_obj.number else: previous_prompt_number = 0 else: previous_prompt_number = content['execution_count'] if self._previous_prompt_obj and \ self._previous_prompt_obj.number != previous_prompt_number: block = self._previous_prompt_obj.block # Make sure the prompt block has not been erased. if block.isValid() and block.text(): # Remove the old prompt and insert a new prompt. cursor = QtGui.QTextCursor(block) cursor.movePosition(QtGui.QTextCursor.Right, QtGui.QTextCursor.KeepAnchor, self._previous_prompt_obj.length) prompt = self._make_in_prompt(previous_prompt_number) self._prompt = self._insert_html_fetching_plain_text( cursor, prompt) # When the HTML is inserted, Qt blows away the syntax # highlighting for the line, so we need to rehighlight it. self._highlighter.rehighlightBlock(cursor.block()) self._previous_prompt_obj = None # Show a new prompt with the kernel's estimated prompt number. self._show_interpreter_prompt(previous_prompt_number + 1) #--------------------------------------------------------------------------- # 'IPythonWidget' interface #--------------------------------------------------------------------------- def set_default_style(self, colors='lightbg'): """ Sets the widget style to the class defaults. Parameters ---------- colors : str, optional (default lightbg) Whether to use the default IPython light background or dark background or B&W style. """ colors = colors.lower() if colors == 'lightbg': self.style_sheet = styles.default_light_style_sheet self.syntax_style = styles.default_light_syntax_style elif colors == 'linux': self.style_sheet = styles.default_dark_style_sheet self.syntax_style = styles.default_dark_syntax_style elif colors == 'nocolor': self.style_sheet = styles.default_bw_style_sheet self.syntax_style = styles.default_bw_syntax_style else: raise KeyError("No such color scheme: %s" % colors) #--------------------------------------------------------------------------- # 'IPythonWidget' protected interface #--------------------------------------------------------------------------- def _edit(self, filename, line=None): """ Opens a Python script for editing. Parameters ---------- filename : str A path to a local system file. line : int, optional A line of interest in the file. """ if self.custom_edit: self.custom_edit_requested.emit(filename, line) elif not self.editor: self._append_plain_text( 'No default editor available.\n' 'Specify a GUI text editor in the `IPythonWidget.editor` ' 'configurable to enable the %edit magic') else: try: filename = '"%s"' % filename if line and self.editor_line: command = self.editor_line.format(filename=filename, line=line) else: try: command = self.editor.format() except KeyError: command = self.editor.format(filename=filename) else: command += ' ' + filename except KeyError: self._append_plain_text('Invalid editor command.\n') else: try: Popen(command, shell=True) except OSError: msg = 'Opening editor with command "%s" failed.\n' self._append_plain_text(msg % command) def _make_in_prompt(self, number): """ Given a prompt number, returns an HTML In prompt. """ try: body = self.in_prompt % number except TypeError: # allow in_prompt to leave out number, e.g. '>>> ' body = self.in_prompt return '<span class="in-prompt">%s</span>' % body def _make_continuation_prompt(self, prompt): """ Given a plain text version of an In prompt, returns an HTML continuation prompt. """ end_chars = '...: ' space_count = len(prompt.lstrip('\n')) - len(end_chars) body = ' ' * space_count + end_chars return '<span class="in-prompt">%s</span>' % body def _make_out_prompt(self, number): """ Given a prompt number, returns an HTML Out prompt. """ body = self.out_prompt % number return '<span class="out-prompt">%s</span>' % body #------ Payload handlers -------------------------------------------------- # Payload handlers with a generic interface: each takes the opaque payload # dict, unpacks it and calls the underlying functions with the necessary # arguments. def _handle_payload_edit(self, item): self._edit(item['filename'], item['line_number']) def _handle_payload_exit(self, item): self._keep_kernel_on_exit = item['keepkernel'] self.exit_requested.emit(self) def _handle_payload_next_input(self, item): self.input_buffer = item['text'] def _handle_payload_page(self, item): # Since the plain text widget supports only a very small subset of HTML # and we have no control over the HTML source, we only page HTML # payloads in the rich text widget. if item['html'] and self.kind == 'rich': self._page(item['html'], html=True) else: self._page(item['text'], html=False) #------ Trait change handlers -------------------------------------------- def _style_sheet_changed(self): """ Set the style sheets of the underlying widgets. """ self.setStyleSheet(self.style_sheet) if self._control is not None: self._control.document().setDefaultStyleSheet(self.style_sheet) bg_color = self._control.palette().window().color() self._ansi_processor.set_background_color(bg_color) if self._page_control is not None: self._page_control.document().setDefaultStyleSheet( self.style_sheet) def _syntax_style_changed(self): """ Set the style for the syntax highlighter. """ if self._highlighter is None: # ignore premature calls return if self.syntax_style: self._highlighter.set_style(self.syntax_style) else: self._highlighter.set_style_sheet(self.style_sheet) #------ Trait default initializers ----------------------------------------- def _banner_default(self): from IPython.core.usage import default_gui_banner return default_gui_banner
def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False, script=False): """ Execute notebook & return the global environment that results from execution. TODO: write a note about the injection of check_results If ignore_errors is True, exceptions are swallowed. secret contains random digits so check_results and check are not easily modifiable nb is passed in as a dictionary that's a parsed ipynb file """ with hide_outputs(): if initial_env: global_env = initial_env.copy() else: global_env = {} source = "" if script: try: exec(nb, global_env) source += nb except: if not ignore_errors: raise else: # Before rewriting AST, find cells of code that generate errors. # One round of execution is done beforehand to mimic the Jupyter notebook style of running # (e.g. code runs up to the point of execution). # The reason this is workaround is introduced is because once the # source code is parsed into an AST, there is no sense of local cells for cell in nb['cells']: if cell['cell_type'] == 'code': # transform the input to executable Python # FIXME: use appropriate IPython functions here isp = IPythonInputSplitter(line_input_checker=False) try: code_lines = [] cell_source_lines = cell['source'] source_is_str_bool = False if isinstance(cell_source_lines, str): source_is_str_bool = True cell_source_lines = cell_source_lines.split('\n') for line in cell_source_lines: # Filter out ipython magic commands # Filter out interact widget if not line.startswith('%'): if "interact(" not in line: code_lines.append(line) if source_is_str_bool: code_lines.append('\n') cell_source = isp.transform_cell(''.join(code_lines)) exec(cell_source, global_env) source += cell_source except: if not ignore_errors: raise tree = ast.parse(source) if find_check_assignment(tree) or find_check_definition(tree): # an empty global_env will fail all the tests return global_env # wrap check(..) calls into a check_results_X.append(check(..)) transformer = CheckCallWrapper(secret) tree = transformer.visit(tree) ast.fix_missing_locations(tree) cleaned_source = compile(tree, filename="nb-ast", mode="exec") try: with open(os.devnull, 'w') as f, redirect_stdout(f), redirect_stderr(f): exec(cleaned_source, global_env) except: if not ignore_errors: raise return global_env
class PyDevFrontEnd: def __init__(self, pydev_host, pydev_client_port, *args, **kwarg): # Create and initialize our IPython instance. self.ipython = PyDevTerminalInteractiveShell.instance() # Back channel to PyDev to open editors (in the future other # info may go back this way. This is the same channel that is # used to get stdin, see StdIn in pydev_console_utils) self.ipython.set_hook('editor', create_editor_hook(pydev_host, pydev_client_port)) # Create an input splitter to handle input separation self.input_splitter = IPythonInputSplitter() # Display the IPython banner, this has version info and # help info self.ipython.show_banner() def complete(self, string): return self.ipython.complete(None, line=string) def getCompletions(self, text, act_tok): # Get completions from IPython and from PyDev and merge the results # IPython only gives context free list of completions, while PyDev # gives detailed information about completions. try: TYPE_IPYTHON = '11' TYPE_IPYTHON_MAGIC = '12' _line, ipython_completions = self.complete(text) from _pydev_completer import Completer completer = Completer(self.getNamespace(), None) ret = completer.complete(act_tok) append = ret.append ip = self.ipython pydev_completions = set([f[0] for f in ret]) for ipython_completion in ipython_completions: if ipython_completion not in pydev_completions: pydev_completions.add(ipython_completion) inf = ip.object_inspect(ipython_completion) if inf['type_name'] == 'Magic function': pydev_type = TYPE_IPYTHON_MAGIC else: pydev_type = TYPE_IPYTHON pydev_doc = inf['docstring'] if pydev_doc is None: pydev_doc = '' append((ipython_completion, pydev_doc, '', pydev_type)) return ret except: import traceback;traceback.print_exc() return [] def getNamespace(self): return self.ipython.user_ns def addExec(self, line): self.input_splitter.push(line) if not self.input_splitter.push_accepts_more(): self.ipython.run_cell(self.input_splitter.source_reset(), store_history=True) return False else: return True
class PyDevFrontEnd: def __init__(self, pydev_host, pydev_client_port, exec_queue, *args, **kwarg): self.exec_queue = exec_queue self._curr_exec_line = 0 # Store certain global objects that IPython modifies _displayhook = sys.displayhook _excepthook = sys.excepthook # Create and initialize our IPython instance. self.ipython = PyDevTerminalInteractiveShell.instance() # Back channel to PyDev to open editors (in the future other # info may go back this way. This is the same channel that is # used to get stdin, see StdIn in pydev_console_utils) self.ipython.set_hook('editor', create_editor_hook(pydev_host, pydev_client_port)) # Create an input splitter to handle input separation self.input_splitter = IPythonInputSplitter() # Display the IPython banner, this has version info and # help info self.ipython.show_banner() # IPython is ready, now clean up some global state... # Deactivate the various python system hooks added by ipython for # interactive convenience so we don't confuse the doctest system sys.displayhook = _displayhook sys.excepthook = _excepthook # So that ipython magics and aliases can be doctested (they work by making # a call into a global _ip object). Also make the top-level get_ipython # now return this without recursively calling here again. try: import __builtin__ except: import builtins as __builtin__ __builtin__._ip = self.ipython __builtin__.get_ipython = self.ipython.get_ipython # We want to print to stdout/err as usual. io.stdout = original_stdout io.stderr = original_stderr def complete(self, string): return self.ipython.complete(None, line=string) def getCompletions(self, text, act_tok): # Get completions from IPython and from PyDev and merge the results # IPython only gives context free list of completions, while PyDev # gives detailed information about completions. try: TYPE_IPYTHON = '11' TYPE_IPYTHON_MAGIC = '12' _line, ipython_completions = self.complete(text) from _pydev_completer import Completer completer = Completer(self.getNamespace(), None) ret = completer.complete(act_tok) append = ret.append ip = self.ipython pydev_completions = set([f[0] for f in ret]) for ipython_completion in ipython_completions: if ipython_completion not in pydev_completions: pydev_completions.add(ipython_completion) inf = ip.object_inspect(ipython_completion) if inf['type_name'] == 'Magic function': pydev_type = TYPE_IPYTHON_MAGIC else: pydev_type = TYPE_IPYTHON pydev_doc = inf['docstring'] if pydev_doc is None: pydev_doc = '' append((ipython_completion, pydev_doc, '', pydev_type)) return ret except: import traceback;traceback.print_exc() return [] def interrupt(self): self.input_splitter.reset() def getNamespace(self): return self.ipython.user_ns def addExec(self, line): self.input_splitter.push(line) if not self.input_splitter.push_accepts_more(): self.exec_queue.put(partial(self.ipython.run_cell, self.input_splitter.source_reset(), store_history=True)) return False else: return True
import secrets import sys import warnings from abc import ABC from ast import AST from enum import Enum from textwrap import dedent, indent from typing import ClassVar, Mapping, Optional, Pattern, Sequence with warnings.catch_warnings(): # see https://github.com/nbQA-dev/nbQA/issues/459 warnings.filterwarnings("ignore", category=DeprecationWarning) from IPython.core.inputsplitter import IPythonInputSplitter INPUT_SPLITTER = IPythonInputSplitter(line_input_checker=False) COMMANDS_WITH_STRING_TOKEN = {"flake8"} class IPythonMagicType(Enum): """Enumeration representing various types of IPython magics.""" SHELL = 0 HELP = 1 LINE = 2 CELL = 3 NO_MAGIC = 4 class MagicHandler(ABC):
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin): """ A Qt frontend for a generic Python kernel. """ # The text to show when the kernel is (re)started. banner = Unicode(config=True) # An option and corresponding signal for overriding the default kernel # interrupt behavior. custom_interrupt = Bool(False) custom_interrupt_requested = QtCore.Signal() # An option and corresponding signals for overriding the default kernel # restart behavior. custom_restart = Bool(False) custom_restart_kernel_died = QtCore.Signal(float) custom_restart_requested = QtCore.Signal() # Whether to automatically show calltips on open-parentheses. enable_calltips = Bool( True, config=True, help="Whether to draw information calltips on open-parentheses.") clear_on_kernel_restart = Bool( True, config=True, help="Whether to clear the console when the kernel is restarted") confirm_restart = Bool( True, config=True, help="Whether to ask for user confirmation when restarting kernel") lexer_class = DottedObjectName(config=True, help="The pygments lexer class to use.") def _lexer_class_changed(self, name, old, new): lexer_class = import_item(new) self.lexer = lexer_class() def _lexer_class_default(self): if py3compat.PY3: return 'pygments.lexers.Python3Lexer' else: return 'pygments.lexers.PythonLexer' lexer = Any() def _lexer_default(self): lexer_class = import_item(self.lexer_class) return lexer_class() # Emitted when a user visible 'execute_request' has been submitted to the # kernel from the FrontendWidget. Contains the code to be executed. executing = QtCore.Signal(object) # Emitted when a user-visible 'execute_reply' has been received from the # kernel and processed by the FrontendWidget. Contains the response message. executed = QtCore.Signal(object) # Emitted when an exit request has been received from the kernel. exit_requested = QtCore.Signal(object) # Protected class variables. _prompt_transformer = IPythonInputSplitter( physical_line_transforms=[classic_prompt()], logical_line_transforms=[], python_line_transforms=[], ) _CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos']) _CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos']) _ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind']) _input_splitter_class = InputSplitter _local_kernel = False _highlighter = Instance(FrontendHighlighter) #--------------------------------------------------------------------------- # 'object' interface #--------------------------------------------------------------------------- def __init__(self, *args, **kw): super(FrontendWidget, self).__init__(*args, **kw) # FIXME: remove this when PySide min version is updated past 1.0.7 # forcefully disable calltips if PySide is < 1.0.7, because they crash if qt.QT_API == qt.QT_API_PYSIDE: import PySide if PySide.__version_info__ < (1, 0, 7): self.log.warn( "PySide %s < 1.0.7 detected, disabling calltips" % PySide.__version__) self.enable_calltips = False # FrontendWidget protected variables. self._bracket_matcher = BracketMatcher(self._control) self._call_tip_widget = CallTipWidget(self._control) self._completion_lexer = CompletionLexer(self.lexer) self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None) self._hidden = False self._highlighter = FrontendHighlighter(self, lexer=self.lexer) self._input_splitter = self._input_splitter_class() self._kernel_manager = None self._kernel_client = None self._request_info = {} self._request_info['execute'] = {} self._callback_dict = {} # Configure the ConsoleWidget. self.tab_width = 4 self._set_continuation_prompt('... ') # Configure the CallTipWidget. self._call_tip_widget.setFont(self.font) self.font_changed.connect(self._call_tip_widget.setFont) # Configure actions. action = self._copy_raw_action key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C action.setEnabled(False) action.setShortcut(QtGui.QKeySequence(key)) action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut) action.triggered.connect(self.copy_raw) self.copy_available.connect(action.setEnabled) self.addAction(action) # Connect signal handlers. document = self._control.document() document.contentsChange.connect(self._document_contents_change) # Set flag for whether we are connected via localhost. self._local_kernel = kw.get('local_kernel', FrontendWidget._local_kernel) #--------------------------------------------------------------------------- # 'ConsoleWidget' public interface #--------------------------------------------------------------------------- def copy(self): """ Copy the currently selected text to the clipboard, removing prompts. """ if self._page_control is not None and self._page_control.hasFocus(): self._page_control.copy() elif self._control.hasFocus(): text = self._control.textCursor().selection().toPlainText() if text: text = self._prompt_transformer.transform_cell(text) QtGui.QApplication.clipboard().setText(text) else: self.log.debug("frontend widget : unknown copy target") #--------------------------------------------------------------------------- # 'ConsoleWidget' abstract interface #--------------------------------------------------------------------------- def _is_complete(self, source, interactive): """ Returns whether 'source' can be completely processed and a new prompt created. When triggered by an Enter/Return key press, 'interactive' is True; otherwise, it is False. """ self._input_splitter.reset() complete = self._input_splitter.push(source) if interactive: complete = not self._input_splitter.push_accepts_more() return complete def _execute(self, source, hidden): """ Execute 'source'. If 'hidden', do not show any output. See parent class :meth:`execute` docstring for full details. """ msg_id = self.kernel_client.execute(source, hidden) self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'user') self._hidden = hidden if not hidden: self.executing.emit(source) def _prompt_started_hook(self): """ Called immediately after a new prompt is displayed. """ if not self._reading: self._highlighter.highlighting_on = True def _prompt_finished_hook(self): """ Called immediately after a prompt is finished, i.e. when some input will be processed and a new prompt displayed. """ # Flush all state from the input splitter so the next round of # reading input starts with a clean buffer. self._input_splitter.reset() if not self._reading: self._highlighter.highlighting_on = False def _tab_pressed(self): """ Called when the tab key is pressed. Returns whether to continue processing the event. """ # Perform tab completion if: # 1) The cursor is in the input buffer. # 2) There is a non-whitespace character before the cursor. text = self._get_input_buffer_cursor_line() if text is None: return False complete = bool(text[:self._get_input_buffer_cursor_column()].strip()) if complete: self._complete() return not complete #--------------------------------------------------------------------------- # 'ConsoleWidget' protected interface #--------------------------------------------------------------------------- def _context_menu_make(self, pos): """ Reimplemented to add an action for raw copy. """ menu = super(FrontendWidget, self)._context_menu_make(pos) for before_action in menu.actions(): if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \ QtGui.QKeySequence.ExactMatch: menu.insertAction(before_action, self._copy_raw_action) break return menu def request_interrupt_kernel(self): if self._executing: self.interrupt_kernel() def request_restart_kernel(self): message = 'Are you sure you want to restart the kernel?' self.restart_kernel(message, now=False) def _event_filter_console_keypress(self, event): """ Reimplemented for execution interruption and smart backspace. """ key = event.key() if self._control_key_down(event.modifiers(), include_command=False): if key == QtCore.Qt.Key_C and self._executing: self.request_interrupt_kernel() return True elif key == QtCore.Qt.Key_Period: self.request_restart_kernel() return True elif not event.modifiers() & QtCore.Qt.AltModifier: # Smart backspace: remove four characters in one backspace if: # 1) everything left of the cursor is whitespace # 2) the four characters immediately left of the cursor are spaces if key == QtCore.Qt.Key_Backspace: col = self._get_input_buffer_cursor_column() cursor = self._control.textCursor() if col > 3 and not cursor.hasSelection(): text = self._get_input_buffer_cursor_line()[:col] if text.endswith(' ') and not text.strip(): cursor.movePosition(QtGui.QTextCursor.Left, QtGui.QTextCursor.KeepAnchor, 4) cursor.removeSelectedText() return True return super(FrontendWidget, self)._event_filter_console_keypress(event) def _insert_continuation_prompt(self, cursor): """ Reimplemented for auto-indentation. """ super(FrontendWidget, self)._insert_continuation_prompt(cursor) cursor.insertText(' ' * self._input_splitter.indent_spaces) #--------------------------------------------------------------------------- # 'BaseFrontendMixin' abstract interface #--------------------------------------------------------------------------- def _handle_complete_reply(self, rep): """ Handle replies for tab completion. """ self.log.debug("complete: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('complete') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): text = '.'.join(self._get_context()) cursor.movePosition(QtGui.QTextCursor.Left, n=len(text)) self._complete_with_items(cursor, rep['content']['matches']) def _silent_exec_callback(self, expr, callback): """Silently execute `expr` in the kernel and call `callback` with reply the `expr` is evaluated silently in the kernel (without) output in the frontend. Call `callback` with the `repr <http://docs.python.org/library/functions.html#repr> `_ as first argument Parameters ---------- expr : string valid string to be executed by the kernel. callback : function function accepting one argument, as a string. The string will be the `repr` of the result of evaluating `expr` The `callback` is called with the `repr()` of the result of `expr` as first argument. To get the object, do `eval()` on the passed value. See Also -------- _handle_exec_callback : private method, deal with calling callback with reply """ # generate uuid, which would be used as an indication of whether or # not the unique request originated from here (can use msg id ?) local_uuid = str(uuid.uuid1()) msg_id = self.kernel_client.execute( '', silent=True, user_expressions={local_uuid: expr}) self._callback_dict[local_uuid] = callback self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'silent_exec_callback') def _handle_exec_callback(self, msg): """Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback`` Parameters ---------- msg : raw message send by the kernel containing an `user_expressions` and having a 'silent_exec_callback' kind. Notes ----- This function will look for a `callback` associated with the corresponding message id. Association has been made by `_silent_exec_callback`. `callback` is then called with the `repr()` of the value of corresponding `user_expressions` as argument. `callback` is then removed from the known list so that any message coming again with the same id won't trigger it. """ user_exp = msg['content'].get('user_expressions') if not user_exp: return for expression in user_exp: if expression in self._callback_dict: self._callback_dict.pop(expression)(user_exp[expression]) def _handle_execute_reply(self, msg): """ Handles replies for code execution. """ self.log.debug("execute: %s", msg.get('content', '')) msg_id = msg['parent_header']['msg_id'] info = self._request_info['execute'].get(msg_id) # unset reading flag, because if execute finished, raw_input can't # still be pending. self._reading = False if info and info.kind == 'user' and not self._hidden: # Make sure that all output from the SUB channel has been processed # before writing a new prompt. self.kernel_client.iopub_channel.flush() # Reset the ANSI style information to prevent bad text in stdout # from messing up our colors. We're not a true terminal so we're # allowed to do this. if self.ansi_codes: self._ansi_processor.reset_sgr() content = msg['content'] status = content['status'] if status == 'ok': self._process_execute_ok(msg) elif status == 'error': self._process_execute_error(msg) elif status == 'aborted': self._process_execute_abort(msg) self._show_interpreter_prompt_for_reply(msg) self.executed.emit(msg) self._request_info['execute'].pop(msg_id) elif info and info.kind == 'silent_exec_callback' and not self._hidden: self._handle_exec_callback(msg) self._request_info['execute'].pop(msg_id) else: super(FrontendWidget, self)._handle_execute_reply(msg) def _handle_input_request(self, msg): """ Handle requests for raw_input. """ self.log.debug("input: %s", msg.get('content', '')) if self._hidden: raise RuntimeError( 'Request for raw input during hidden execution.') # Make sure that all output from the SUB channel has been processed # before entering readline mode. self.kernel_client.iopub_channel.flush() def callback(line): self.kernel_client.stdin_channel.input(line) if self._reading: self.log.debug( "Got second input request, assuming first was interrupted.") self._reading = False self._readline(msg['content']['prompt'], callback=callback) def _kernel_restarted_message(self, died=True): msg = "Kernel died, restarting" if died else "Kernel restarting" self._append_html("<br>%s<hr><br>" % msg, before_prompt=False) def _handle_kernel_died(self, since_last_heartbeat): """Handle the kernel's death (if we do not own the kernel). """ self.log.warn("kernel died: %s", since_last_heartbeat) if self.custom_restart: self.custom_restart_kernel_died.emit(since_last_heartbeat) else: self._kernel_restarted_message(died=True) self.reset() def _handle_kernel_restarted(self, died=True): """Notice that the autorestarter restarted the kernel. There's nothing to do but show a message. """ self.log.warn("kernel restarted") self._kernel_restarted_message(died=died) self.reset() def _handle_object_info_reply(self, rep): """ Handle replies for call tips. """ self.log.debug("oinfo: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('call_tip') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): # Get the information for a call tip. For now we format the call # line as string, later we can pass False to format_call and # syntax-highlight it ourselves for nicer formatting in the # calltip. content = rep['content'] # if this is from pykernel, 'docstring' will be the only key if content.get('ismagic', False): # Don't generate a call-tip for magics. Ideally, we should # generate a tooltip, but not on ( like we do for actual # callables. call_info, doc = None, None else: call_info, doc = call_tip(content, format_call=True) if call_info or doc: self._call_tip_widget.show_call_info(call_info, doc) def _handle_pyout(self, msg): """ Handle display hook output. """ self.log.debug("pyout: %s", msg.get('content', '')) if not self._hidden and self._is_from_this_session(msg): text = msg['content']['data'] self._append_plain_text(text + '\n', before_prompt=True) def _handle_stream(self, msg): """ Handle stdout, stderr, and stdin. """ self.log.debug("stream: %s", msg.get('content', '')) if not self._hidden and self._is_from_this_session(msg): # Most consoles treat tabs as being 8 space characters. Convert tabs # to spaces so that output looks as expected regardless of this # widget's tab width. text = msg['content']['data'].expandtabs(8) self._append_plain_text(text, before_prompt=True) self._control.moveCursor(QtGui.QTextCursor.End) def _handle_shutdown_reply(self, msg): """ Handle shutdown signal, only if from other console. """ self.log.warn("shutdown: %s", msg.get('content', '')) restart = msg.get('content', {}).get('restart', False) if not self._hidden and not self._is_from_this_session(msg): # got shutdown reply, request came from session other than ours if restart: # someone restarted the kernel, handle it self._handle_kernel_restarted(died=False) else: # kernel was shutdown permanently # this triggers exit_requested if the kernel was local, # and a dialog if the kernel was remote, # so we don't suddenly clear the qtconsole without asking. if self._local_kernel: self.exit_requested.emit(self) else: title = self.window().windowTitle() reply = QtGui.QMessageBox.question( self, title, "Kernel has been shutdown permanently. " "Close the Console?", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No) if reply == QtGui.QMessageBox.Yes: self.exit_requested.emit(self) def _handle_status(self, msg): """Handle status message""" # This is where a busy/idle indicator would be triggered, # when we make one. state = msg['content'].get('execution_state', '') if state == 'starting': # kernel started while we were running if self._executing: self._handle_kernel_restarted(died=True) elif state == 'idle': pass elif state == 'busy': pass def _started_channels(self): """ Called when the KernelManager channels have started listening or when the frontend is assigned an already listening KernelManager. """ self.reset(clear=True) #--------------------------------------------------------------------------- # 'FrontendWidget' public interface #--------------------------------------------------------------------------- def copy_raw(self): """ Copy the currently selected text to the clipboard without attempting to remove prompts or otherwise alter the text. """ self._control.copy() def execute_file(self, path, hidden=False): """ Attempts to execute file with 'path'. If 'hidden', no output is shown. """ self.execute('execfile(%r)' % path, hidden=hidden) def interrupt_kernel(self): """ Attempts to interrupt the running kernel. Also unsets _reading flag, to avoid runtime errors if raw_input is called again. """ if self.custom_interrupt: self._reading = False self.custom_interrupt_requested.emit() elif self.kernel_manager: self._reading = False self.kernel_manager.interrupt_kernel() else: self._append_plain_text( 'Cannot interrupt a kernel I did not start.\n') def reset(self, clear=False): """ Resets the widget to its initial state if ``clear`` parameter is True, otherwise prints a visual indication of the fact that the kernel restarted, but does not clear the traces from previous usage of the kernel before it was restarted. With ``clear=True``, it is similar to ``%clear``, but also re-writes the banner and aborts execution if necessary. """ if self._executing: self._executing = False self._request_info['execute'] = {} self._reading = False self._highlighter.highlighting_on = False if clear: self._control.clear() self._append_plain_text(self.banner) # update output marker for stdout/stderr, so that startup # messages appear after banner: self._append_before_prompt_pos = self._get_cursor().position() self._show_interpreter_prompt() def restart_kernel(self, message, now=False): """ Attempts to restart the running kernel. """ # FIXME: now should be configurable via a checkbox in the dialog. Right # now at least the heartbeat path sets it to True and the manual restart # to False. But those should just be the pre-selected states of a # checkbox that the user could override if so desired. But I don't know # enough Qt to go implementing the checkbox now. if self.custom_restart: self.custom_restart_requested.emit() return if self.kernel_manager: # Pause the heart beat channel to prevent further warnings. self.kernel_client.hb_channel.pause() # Prompt the user to restart the kernel. Un-pause the heartbeat if # they decline. (If they accept, the heartbeat will be un-paused # automatically when the kernel is restarted.) if self.confirm_restart: buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No result = QtGui.QMessageBox.question(self, 'Restart kernel?', message, buttons) do_restart = result == QtGui.QMessageBox.Yes else: # confirm_restart is False, so we don't need to ask user # anything, just do the restart do_restart = True if do_restart: try: self.kernel_manager.restart_kernel(now=now) except RuntimeError as e: self._append_plain_text('Error restarting kernel: %s\n' % e, before_prompt=True) else: self._append_html( "<br>Restarting kernel...\n<hr><br>", before_prompt=True, ) else: self.kernel_client.hb_channel.unpause() else: self._append_plain_text( 'Cannot restart a Kernel I did not start\n', before_prompt=True) #--------------------------------------------------------------------------- # 'FrontendWidget' protected interface #--------------------------------------------------------------------------- def _call_tip(self): """ Shows a call tip, if appropriate, at the current cursor location. """ # Decide if it makes sense to show a call tip if not self.enable_calltips: return False cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.Left) if cursor.document().characterAt(cursor.position()) != '(': return False context = self._get_context(cursor) if not context: return False # Send the metadata request to the kernel name = '.'.join(context) msg_id = self.kernel_client.object_info(name) pos = self._get_cursor().position() self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos) return True def _complete(self): """ Performs completion at the current cursor location. """ context = self._get_context() if context: # Send the completion request to the kernel msg_id = self.kernel_client.complete( '.'.join(context), # text self._get_input_buffer_cursor_line(), # line self._get_input_buffer_cursor_column(), # cursor_pos self.input_buffer) # block pos = self._get_cursor().position() info = self._CompletionRequest(msg_id, pos) self._request_info['complete'] = info def _get_context(self, cursor=None): """ Gets the context for the specified cursor (or the current cursor if none is specified). """ if cursor is None: cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.StartOfBlock, QtGui.QTextCursor.KeepAnchor) text = cursor.selection().toPlainText() return self._completion_lexer.get_context(text) def _process_execute_abort(self, msg): """ Process a reply for an aborted execution request. """ self._append_plain_text("ERROR: execution aborted\n") def _process_execute_error(self, msg): """ Process a reply for an execution request that resulted in an error. """ content = msg['content'] # If a SystemExit is passed along, this means exit() was called - also # all the ipython %exit magic syntax of '-k' to be used to keep # the kernel running if content['ename'] == 'SystemExit': keepkernel = content['evalue'] == '-k' or content[ 'evalue'] == 'True' self._keep_kernel_on_exit = keepkernel self.exit_requested.emit(self) else: traceback = ''.join(content['traceback']) self._append_plain_text(traceback) def _process_execute_ok(self, msg): """ Process a reply for a successful execution request. """ payload = msg['content']['payload'] for item in payload: if not self._process_execute_payload(item): warning = 'Warning: received unknown payload of type %s' print(warning % repr(item['source'])) def _process_execute_payload(self, item): """ Process a single payload item from the list of payload items in an execution reply. Returns whether the payload was handled. """ # The basic FrontendWidget doesn't handle payloads, as they are a # mechanism for going beyond the standard Python interpreter model. return False def _show_interpreter_prompt(self): """ Shows a prompt for the interpreter. """ self._show_prompt('>>> ') def _show_interpreter_prompt_for_reply(self, msg): """ Shows a prompt for the interpreter given an 'execute_reply' message. """ self._show_interpreter_prompt() #------ Signal handlers ---------------------------------------------------- def _document_contents_change(self, position, removed, added): """ Called whenever the document's content changes. Display a call tip if appropriate. """ # Calculate where the cursor should be *after* the change: position += added document = self._control.document() if position == self._get_cursor().position(): self._call_tip() #------ Trait default initializers ----------------------------------------- def _banner_default(self): """ Returns the standard Python banner. """ banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \ '"license" for more information.' return banner % (sys.version, sys.platform)
def __init__(self, *args, **kwargs): super(IPythonValidator, self).__init__(*args, **kwargs) self.isp = IPythonInputSplitter()
from IPython.core.inputsplitter import IPythonInputSplitter spliter = IPythonInputSplitter() cell = "%time None" print(spliter.transform_cell(cell)) # get_ipython().run_line_magic('time', 'None')
out = isp.transform_cell(raw) self.assertEqual(out.rstrip(), expected.rstrip()) #----------------------------------------------------------------------------- # Main - use as a script, mostly for developer experiments #----------------------------------------------------------------------------- if __name__ == '__main__': # A simple demo for interactive experimentation. This code will not get # picked up by any test suite. from IPython.core.inputsplitter import IPythonInputSplitter # configure here the syntax to use, prompt and whether to autoindent #isp, start_prompt = InputSplitter(), '>>> ' isp, start_prompt = IPythonInputSplitter(), 'In> ' autoindent = True #autoindent = False try: while True: prompt = start_prompt while isp.push_accepts_more(): indent = ' ' * isp.get_indent_spaces() if autoindent: line = indent + input(prompt + indent) else: line = input(prompt) isp.push(line) prompt = '... '
def execute_log(nb, log, secret='secret', initial_env=None, ignore_errors=False, cwd=None, test_dir=None, variables=None): """ Executes a notebook from logged environments and returns the global environment that results from execution Execute notebook & return the global environment that results from execution. If ``ignore_errors`` is ``True``, exceptions are swallowed. ``secret`` contains random digits so ``check_results`` and ``check`` are not easily modifiable. ``nb`` is passed in as a dictionary that's a parsed notebook Args: nb (``dict``): JSON representation of a notebook log (``otter.check.logs.Log``): log from notebook execution secret (``str``, optional): randomly generated integer used to rebind check function initial_env (``str``, optional): name of initial environment ignore_errors (``bool``, optional): whether exceptions should be ignored cwd (``str``, optional): working directory of execution to be appended to ``sys.path`` in grading environment test_dir (``str``, optional): path to directory of tests in grading environment variables (``dict``, optional): map of variable names -> type string to check type of deserialized object to prevent arbitrary code from being put into the environment Results: ``dict``: global environment resulting from executing all code of the input notebook """ with hide_outputs(): if initial_env: global_env = initial_env.copy() else: global_env = {} if test_dir: source = f"import otter\ngrader = otter.Notebook(\"{test_dir}\")\n" else: source = f"import otter\ngrader = otter.Notebook()\n" if cwd: source += f"import sys\nsys.path.append(\"{cwd}\")\n" logged_questions = [] m = mock.mock_open() with mock.patch("otter.Notebook._log_event", m): exec(source, global_env) for cell in nb['cells']: if cell['cell_type'] == 'code': # transform the input to executable Python # FIXME: use appropriate IPython functions here isp = IPythonInputSplitter(line_input_checker=False) code_lines = [] cell_source_lines = cell['source'] source_is_str_bool = False if isinstance(cell_source_lines, str): source_is_str_bool = True cell_source_lines = cell_source_lines.split('\n') # only execute import statements cell_source_lines = [ re.sub(r"^\s+", "", l) for l in cell_source_lines if "import" in l ] for line in cell_source_lines: try: exec(line, global_env) # source += cell_source except: if not ignore_errors: raise for entry in log.question_iterator(): shelf = entry.unshelve(global_env) if variables is not None: for k, v in shelf.items(): full_type = type(v).__module__ + "." + type(v).__name__ if not (k in variables and variables[k] == full_type): del shelf[k] print( f"Found variable of different type than expected: {k}" ) global_env.update(shelf) global_env[f"check_results_{secret}"].append( global_env["grader"].check(entry.question, global_env=global_env)) logged_questions.append(entry.question) print("Questions executed from log: {}".format( ", ".join(logged_questions))) return global_env
def __init__(self, *args, **kwargs): super(IPythonValidator, self).__init__(*args, **kwargs) self.isp = IPythonInputSplitter()
def execute_notebook(nb, secret='secret', initial_env=None, ignore_errors=False, gradescope=False): """ Executes an ipython notebook and return the global environment that results from execution. Execute notebook & return the global environment that results from execution. TODO: write a note about the injection of check_results If ignore_errors is True, exceptions are swallowed. secret contains random digits so check_results and check are not easily modifiable nb is passed in as a dictionary that's a parsed ipynb file Args: nb (dict of str: str): json representation of ipython notebook secret (str, optional): randomly generated integer used to rebind check function initial_env (str, optional): name of initial environment ignore_errors (bool, optional): whether exceptions should be ignored Results: dict of str: object: global environment resulting from executing all code of the input notebook """ with hide_outputs(): if initial_env: global_env = initial_env.copy() else: global_env = {} source = "" if gradescope: source = "import sys\nsys.path.append(\"/autograder/submission\")\n" # Before rewriting AST, find cells of code that generate errors. # One round of execution is done beforehand to mimic the Jupyter notebook style of running # (e.g. code runs up to the point of execution). # The reason this is workaround is introduced is because once the # source code is parsed into an AST, there is no sense of local cells for cell in nb['cells']: if cell['cell_type'] == 'code': # transform the input to executable Python # FIXME: use appropriate IPython functions here isp = IPythonInputSplitter(line_input_checker=False) try: code_lines = [] cell_source_lines = cell['source'] source_is_str_bool = False if isinstance(cell_source_lines, str): source_is_str_bool = True cell_source_lines = cell_source_lines.split('\n') for line in cell_source_lines: # Filter out ipython magic commands # Filter out interact widget if not line.startswith('%'): if "interact(" not in line and not re.search( r"otter\.Notebook\(.*?\)", line): code_lines.append(line) if source_is_str_bool: code_lines.append('\n') elif re.search(r"otter\.Notebook\(.*?\)", line): if gradescope: line = re.sub( r"otter\.Notebook\(.*?\)", "otter.Notebook(\"/autograder/submission/tests\")", line) else: line = re.sub( r"otter\.Notebook\(.*?\)", "otter.Notebook(\"/home/tests\")", line) code_lines.append(line) if source_is_str_bool: code_lines.append('\n') cell_source = isp.transform_cell(''.join(code_lines)) # patch otter.Notebook.export so that we don't create PDFs in notebooks m = mock.mock_open() with mock.patch('otter.Notebook.export', m): exec(source + cell_source, global_env) source += cell_source except: if not ignore_errors: raise tree = ast.parse(source) # # CODE BELOW COMMENTED OUT BECAUSE the only check function is within the Notebook class # if find_check_assignment(tree) or find_check_definition(tree): # # an empty global_env will fail all the tests # return global_env # wrap check(..) calls into a check_results_X.append(check(..)) transformer = CheckCallWrapper(secret) tree = transformer.visit(tree) ast.fix_missing_locations(tree) cleaned_source = compile(tree, filename="nb-ast", mode="exec") try: with open(os.devnull, 'w') as f, redirect_stdout(f), redirect_stderr(f): # patch otter.Notebook.export so that we don't create PDFs in notebooks m = mock.mock_open() with mock.patch('otter.Notebook.export', m): exec(cleaned_source, global_env) except: if not ignore_errors: raise return global_env