def run_ast_nodes(self,
nodelist: ListType[AST],
cell_name: str,
interactivity='last_expr',
compiler=compile,
result=None):
if not nodelist:
return
try:
has_exc = self.transfer_nodelist(nodelist, result, cell_name)
if result.result is not None:
out_result = "{0}".format(result.result)
try:
if type(result.result) == str:
raise
mod = ast.Interactive([ast.parse(out_result).body[0]])
exec(compiler(mod, "ev", "single"))
except:
out_result = r"'{0}'".format(
str(result.result).replace("\\",
"\\\\").replace("'", "\\'"))
mod = ast.Interactive([ast.parse(out_result).body[0]])
exec(compiler(mod, "ev", "single"))
if has_exc:
return True
except:
if result:
result.error_before_exec = sys.exc_info()[1]
self.showtraceback()
return True
return False
def extract_functions(codestring, filename='tmp'):
"""
parse the code *codestring* and detect what are the functions defined inside
:return: dict name -> code
"""
# TODO: use ast.iter_child_nodes instead of walk to know nested level
# For example, currently this code fails because indent level is wrong
# if True:
# def f():
# print('hello')
# This code fails because "return is outside of function"
# def f():
# return 1
# TODO: support IPython %magic
funcs = {}
r = ast_parse(codestring)
for statement in ast.walk(r):
if isinstance(statement, ast.FunctionDef):
wrapped = ast.Interactive(body=statement.body)
try:
code = compile(wrapped, filename, 'single')
except SyntaxError:
logger.debug("Parsing code %s not yet supported" %
statement.name)
else:
funcs[statement.name] = code
return funcs
def _exec(excode):
"""this code is boiled down from
https://github.com/ipython/ipython/blob/master/IPython/core/interactiveshell.py
it makes sure that expressions on the
last line of a nbplots directive are evaluated and printed
"""
import ast
if sys.version_info > (3, 8):
from ast import Module
else:
# mock the new API, ignore second argument
# see https://github.com/ipython/ipython/issues/11590
from ast import Module as OriginalModule
Module = lambda nodelist, type_ignores: OriginalModule(nodelist)
nodelist = ast.parse(excode).body
if not nodelist:
return
if isinstance(nodelist[-1], ast.Expr):
to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
else:
to_run_exec, to_run_interactive = nodelist, []
to_run = [(node, 'exec') for node in to_run_exec]
for node in to_run_interactive:
to_run.append((node, 'single'))
for node, mode in to_run:
if mode == 'exec':
mod = Module([node], [])
elif mode == 'single':
mod = ast.Interactive([node])
six.exec_(compile(mod, '<dummyfile>', mode), ns)
def execute(code):
try:
code = ast.parse(code)
to_run_exec, to_run_single = code.body[:-1], code.body[-1:]
for node in to_run_exec:
mod = ast.Module([node])
code = compile(mod, '<stdin>', 'exec')
exec code in global_dict
for node in to_run_single:
mod = ast.Interactive([node])
code = compile(mod, '<stdin>', 'single')
exec code in global_dict
except:
return execute_reply_error(*sys.exc_info())
stdout = fake_stdout.getvalue()
fake_stdout.truncate(0)
stderr = fake_stderr.getvalue()
fake_stderr.truncate(0)
output = ''
if stdout:
output += stdout
if stderr:
output += stderr
return execute_reply_ok({
'text/plain': output.rstrip(),
})
def eval_block(code, namespace=None, filename="<string>"):
"""
Execute a multi-line block of code in the given namespace.
If the final statement in the code is an expression, return the result of the
expression.
"""
tree = ast.parse(code, filename="<ast>", mode="exec")
if namespace is None:
namespace = {}
if isinstance(tree.body[-1], ast.Expr):
to_exec, to_eval = tree.body[:-1], tree.body[-1:]
else:
to_exec, to_eval = tree.body, []
for node in to_exec:
compiled = compile(ast.Module([node]), filename=filename, mode="exec")
exec(compiled, namespace)
catch_display = _CatchDisplay()
with catch_display:
for node in to_eval:
compiled = compile(ast.Interactive([node]),
filename=filename,
mode="single")
exec(compiled, namespace)
return catch_display.output
def _executer(self):
while True:
code = (yield self._stdout.getvalue())
self._is_exception = self.NO_EXCEPTION
try:
code_tree = ast.parse(code)
exec_code = code_tree.body[:-1]
single_code = code_tree.body[-1:] # last node of code
exec_code_object = compile(ast.Module(exec_code), '<string>',
'exec')
interactive_code_object = compile(ast.Interactive(single_code),
'<string>', 'single')
with redirect_stderr():
exec(exec_code_object, self._globals)
exec(interactive_code_object, self._globals)
except BrighticsCoreException as bce:
raise bce
except BrighticsFunctionException as bfe:
raise bfe
except Exception as e:
self._stdout.write(traceback.format_exc())
self._is_exception = (True,
traceback.format_exception_only(
type(e), e)[-1])
def runsource(self, source, filename="<input>", symbol="single"):
try:
code = self.compile(source, filename, symbol)
except (OverflowError, SyntaxError, ValueError):
code = ""
pass
if code is None:
# This means it's incomplete
return True
try:
tree = ast.parse(source)
required_pkgs = detect_macros(tree)
for p in required_pkgs:
__import__(p)
self.modules.update(sys.modules[p] for p in required_pkgs)
tree = process_ast(tree, self.modules)
tree = ast.Interactive(tree.body)
code = compile(tree, filename, symbol, self.compile.compiler.flags,
1)
except (OverflowError, SyntaxError, ValueError):
# Case 1
self.showsyntaxerror(filename)
# This means there's a syntax error
return False
self.runcode(code)
# This means it was successfully compiled; `runcode` takes care of
# any runtime failures
return False
def execute(code):
try:
to_run_exec, to_run_single = code.body[:-1], code.body[-1:]
for node in to_run_exec:
mod = ast.Module([node])
code = compile(mod, '<stdin>', 'exec')
exec code in global_dict
for node in to_run_single:
mod = ast.Interactive([node])
code = compile(mod, '<stdin>', 'single')
exec code in global_dict
except:
# We don't need to log the exception because we're just executing user
# code and passing the error along.
return execute_reply_error(*sys.exc_info())
stdout = fake_stdout.getvalue()
fake_stdout.truncate(0)
stderr = fake_stderr.getvalue()
fake_stderr.truncate(0)
output = ''
if stdout:
output += stdout
if stderr:
output += stderr
return execute_reply_ok({
'text/plain': output.rstrip(),
})
def exec_code(self, code, tests=[]):
self.prepare_env()
signal('plutoid::code_execution_start').send('plutoid')
exc = None
try:
code_obj = compile(code, 'your-code', 'exec', ast.PyCF_ONLY_AST, 1)
for node in code_obj.body:
single_code_obj = compile(ast.Interactive([node]), 'your-code',
'single')
exec(single_code_obj, self.globals)
for test in tests:
result = 'ok'
test_code_obj = compile(test, 'tests', 'exec')
try:
exec(test_code_obj, self.globals)
except AssertionError:
result = 'not-ok'
signal('plutoid::test_result').send('plutoid',
content=test,
result=result)
except Exception as e:
self.print_exception()
exc = e
finally:
self.revert_env()
signal('plutoid::code_execution_end').send('plutoid')
return exc
def runsource(self, source, filename="<input>", symbol="single"):
try:
code = self.compile(source, filename, symbol)
except (OverflowError, SyntaxError, ValueError):
code = ""
pass
if code is None:
# This means it's incomplete
return True
try:
tree = ast.parse(source)
bindings = detect_macros(tree, '__main__')
for mod, bind in bindings:
self.bindings.append((importlib.import_module(mod), bind))
tree = ModuleExpansionContext(tree, source,
self.bindings).expand_macros()
tree = ast.Interactive(tree.body)
code = compile(tree, filename, symbol, self.compile.compiler.flags,
1)
except (OverflowError, SyntaxError, ValueError):
# Case 1
self.showsyntaxerror(filename)
# This means there's a syntax error
return False
self.runcode(code)
# This means it was successfully compiled; `runcode` takes care of
# any runtime failures
return False
def interpret(code_text):
if not DO_PYTHON:
return "I'm not doing Python.", ""
saved_stdout = sys.stdout
saved_stderr = sys.stderr
output = StringIO.StringIO()
trace = ""
try:
sys.stdout = output
sys.stderr = output
tree = ast.parse(code_text, "<your input>")
code1 = code2 = None
if tree.body and isinstance(tree.body[-1], ast.Expr):
last_line = ast.Interactive(tree.body[-1:])
tree.body = tree.body[:-1]
code2 = compile(last_line, "<your input>", "single")
if tree.body:
code1 = compile(tree, "<your input>", "exec")
if code1:
exec code1 in NOTEBOOK_GLOBALS
if code2:
exec code2 in NOTEBOOK_GLOBALS
except Exception, KeyboardInterrupt:
trace = traceback.format_exc()
def parse_functions(codestring):
"""
Parse the code *codestring* and detect what are the functions defined inside:
- Search *init*, *step*, *animate* and *run*
:return: init, step, animate, run functions (code or False)
"""
exec_funcs = {}
exec_funcs_names = ['init', 'step', 'animate', 'run']
for func_name in exec_funcs_names:
exec_funcs[func_name] = False
r = ast_parse(codestring)
functions_list = [x for x in ast.walk(r) if isinstance(x, ast.FunctionDef)]
for x in functions_list:
if x.name in exec_funcs_names:
wrapped = ast.Interactive(body=x.body)
try:
code = compile(wrapped, 'tmp', 'single')
except:
pass
else:
exec_funcs[x.name] = code
exec_funcs_list = [exec_funcs[func_name] for func_name in exec_funcs_names]
return exec_funcs_list
def sub_parser(self, *args, **kwargs):
if kwargs.get('func'):
sub_parser = FunctionAnalyzer(context=self)
else:
sub_parser = Analyzer(context=self)
sub_parser.indent += 1
temp = ast.Interactive(list(args))
sub_parser.visit(temp)
return sub_parser
def compiler(input, filename, symbol, *args, **kwargs):
return get_ipython().compile(
ast.Interactive(body=shell.transform_ast(
shell.compile.ast_parse(
shell.transform_cell(textwrap.indent(input, " " *
4)))).body),
f"In[{shell.last_execution_result.execution_count}]",
"single",
)
def visit_Interactive(self, node):
"""
Used when code is compiled by interactive console.
Args:
node (ast.AST): ast node.
"""
return ast.Interactive(body=self.block_visit(node.body))
def generate(tree) -> "bytecode":
"""
With the help of `translate()` below, generate Python bytecode for the Python's `eval()`.
https://docs.python.org/3/library/functions.html#compile
"""
return compile(source=ast.fix_missing_locations(
ast.Interactive(body=[wrap(translate(tree))])),
filename="<input>",
mode="single")
def compiler(input, filename, symbol, *args, **kwargs):
nonlocal shell
return shell.compile(
ast.Interactive(body=shell.transform_ast(
shell.compile.ast_parse(
shell.transform_cell(textwrap.indent(input, " " *
4)))).body),
filename,
"single",
)
def _eval(self, node: ast.AST):
try:
code = compile(ast.Expression(node), inspect.getfile(self._module),
"eval")
except TypeError:
exec(
compile(ast.Interactive([node]), inspect.getfile(self._module),
"single"), self._globals, self._locals)
else:
return eval(code, self._globals, self._locals)
def parse(self, code):
self.lexer.input(code) # let code be the input of the lexer
parsedListStatements = self.parser.parse(
lexer=self.lexer
) # this list is p[0], constructed with the production rules
if self.single:
syntaxTree = ast.Interactive(body=parsedListStatements)
else:
syntaxTree = ast.Module(body=parsedListStatements)
ast.fix_missing_locations(
syntaxTree
) # Adds line numbers to the generated program. If the program gets stuck, this tells where the error was.
return syntaxTree
def eval_python(code,
code_dir,
output_dir,
output_base,
filename="<string>",
config=None):
# pylint: disable=exec-used,too-many-arguments
"""
Execute a multi-line block of Python code and copy the generated image files
to specified output directory.
"""
tree = ast.parse(code, filename="<ast>", mode="exec")
if (isinstance(tree.body[-1], ast.Expr) and tree.body[-1].value.func.attr
== "show"): # last statement is `fig.show()` in pygmt
to_exec, to_eval = tree.body[:-1], tree.body[-1:]
else:
to_exec, to_eval = tree.body, []
cwd = os.getcwd()
with environ({
"GMT_END_SHOW": "off",
"GMT_DATADIR": _updated_gmt_datadir(code_dir)
}), tempfile.TemporaryDirectory() as tmpdir:
os.chdir(tmpdir)
if config.gmtplot_gmt_config:
_write_gmt_config(config.gmtplot_gmt_config, cwd=".")
for node in to_exec:
exec(
compile(ast.Module([node], type_ignores=[]),
filename=filename,
mode="exec"))
images = _search_images(tmpdir)
if images:
for image in images:
shutil.move(
image,
Path(output_dir, output_base).with_suffix(image.suffix))
else:
catch_display = _CatchDisplay()
with catch_display:
for node in to_eval:
exec(
compile(ast.Interactive([node]),
filename=filename,
mode="single"))
Path(output_dir, output_base).with_suffix(".png").write_bytes(
catch_display.output.data)
os.chdir(cwd)
return f"{output_base}.*"
def parse(self, code, file_name, interactive=False):
self.lexer.input(code)
tree = self.parser.parse(lexer=self.lexer, debug=False)
if errors:
first_error = None
for line, msg in errors:
if line == -1:
print('{}\t{}'.format(file_name, msg))
else:
print('{}:{}\t{}'.format(file_name, line, msg))
raise SyntaxError
if interactive:
return ast.Interactive(tree)
else:
return ast.Module(tree)
def __exec_then_eval(code):
## processes statements and expressesions separatrly
## returns the value of last expression
block = ast.parse(code, mode="exec")
lv = None
for i in block.body:
if isinstance(i, ast.Expr):
lv = eval(
compile(ast.Expression(i.value), "<string>", mode="eval"),
globals())
else:
lv = None
exec(compile(ast.Interactive([i]), "<string>", mode="single"),
globals())
return lv
def run_source(
self,
source: ta.Union[str, ast.AST],
filename: str = '<input>',
symbol: str = 'single',
) -> bool:
try:
code = self.compile(source, filename, symbol)
except (OverflowError, SyntaxError, ValueError):
# Case 1 (incorrect)
self.show_syntax_error(filename)
return False
if code is None:
# Case 2 (incomplete)
return True
# Case 3 (complete)
try:
node = ast.parse(source)
except (OverflowError, SyntaxError, ValueError):
return True
if isinstance(node, ast.Module) and node.body and isinstance(
node.body[-1], ast.Expr):
expr = node.body[-1]
source = ast.Interactive([
*node.body[:-1],
ast.Assign(
[
ast.Name(
f'_{self._count}',
ast.Store(),
lineno=expr.lineno,
col_offset=expr.col_offset,
)
],
expr.value,
lineno=expr.lineno,
col_offset=expr.col_offset,
)
], )
ast.fix_missing_locations(source)
self._write_count = self._count
code = self.compile(source, filename, symbol)
self.run_code(code)
return False
def execute(self, source):
"""Execute source code inside the space namespace and outer namespace.
Args:
source (str): Source code.
"""
tree = ast.parse(source=source)
interactive_tree = ast.Interactive(body=tree.body)
if sys.version_info > (3, 8):
interactive_tree.type_ignores = tree.type_ignores
compiled_interactive_tree = compile(
source=interactive_tree, filename="<string>", mode="single"
)
exec(compiled_interactive_tree, self._execution_namespace)
def execute(self):
to_run_exec, to_run_single = self.code.body[:-1], self.code.body[-1:]
try:
for node in to_run_exec:
mod = ast.Module([node])
code = compile(mod, '<stdin>', 'exec')
exec code in global_dict
for node in to_run_single:
mod = ast.Interactive([node])
code = compile(mod, '<stdin>', 'single')
exec code in global_dict
except:
# We don't need to log the exception because we're just executing user
# code and passing the error along.
raise ExecutionError(sys.exc_info())
def runsource(self, source, filename="<interactive input>", symbol="single"):
# Special REPL commands.
if source == "macros?":
self.write(format_bindings(self.expander))
return False # complete input
elif source.endswith("??"):
return self.runsource(f'mcpyrate.repl.utils.sourcecode({source[:-2]})')
elif source.endswith("?"):
return self.runsource(f"mcpyrate.repl.utils.doc({source[:-1]})")
try:
code = self.compile(source, filename, symbol)
except (OverflowError, SyntaxError, ValueError):
code = ""
if code is None: # incomplete input
return True
try:
# TODO: If we want to support dialects in the REPL, this is where to do it.
tree = ast.parse(source)
bindings = find_macros(tree, filename=self.expander.filename, reload=True) # macro-imports (this will import the modules)
if bindings:
self._macro_bindings_changed = True
self.expander.bindings.update(bindings)
tree = self.expander.visit(tree)
tree = global_postprocess(tree)
tree = ast.Interactive(tree.body)
code = compile(tree, filename, symbol, self.compile.compiler.flags, 1)
except (OverflowError, SyntaxError, ValueError, MacroExpansionError):
self.showsyntaxerror(filename)
return False # erroneous input
except ModuleNotFoundError as err: # during macro module lookup
# In this case, the standard stack trace is long and points only to our code and the stdlib,
# not the erroneous input that's the actual culprit. Better ignore it, and emulate showsyntaxerror.
# TODO: support sys.excepthook.
self.write(f"{err.__class__.__name__}: {str(err)}\n")
return False # erroneous input
except ImportError as err: # during macro lookup in a successfully imported module
self.write(f"{err.__class__.__name__}: {str(err)}\n")
return False # erroneous input
self.runcode(code)
self._refresh_macro_functions()
return False # Successfully compiled. `runcode` takes care of any runtime failures.
def interpret_code(code):
# # Setting this var lets pip be used from a thread other than its original import
# import threading
# _log_state = threading.local()
# if not hasattr(_log_state, 'indentation'):
# _log_state.indentation = 0
try:
sys.stdout.can_omit = True
# The input is first parsed as ast, then if the last statement
# is an Expr compiled partially in single mode. This means
# that the last statement output is printed, as in the normal
# Python interpreter
components = ast.parse(code).body
# print('components are', components)
# exec all but the last ast component in exec mode
if len(components) > 1:
for component in components[:-1]:
c = compile(ast.Module([component]), '<stdin>', mode='exec')
exec(c, user_locals, user_globals)
# if the last ast component is an Expr, compile in single mode to print it
if isinstance(components[-1], ast.Expr):
c = compile(ast.Interactive([components[-1]]),
'<stdin>',
mode='single')
else:
c = compile(ast.Module([components[-1]]), '<stdin>', mode='exec')
exec(c, user_locals, user_globals)
except KeyboardInterrupt as e:
print('')
traceback.print_exc()
osc.sendMsg(b'/interpreter', [b'keyboard_interrupted'],
port=send_port,
typehint='b')
except Exception as e:
traceback.print_exc()
finally:
sys.stdout.can_omit = False
complete_execution()
def _internal_execute(self, source):
"""Execute given source in the console session.
This is support magic for internal operation of the console
session itself, e.g. for auto-loading macro functions.
The source must be pure Python, i.e. no macros.
The source is NOT added to the session history.
This bypasses `runsource`, so it too can use this function.
"""
source = textwrap.dedent(source)
tree = ast.parse(source)
tree = ast.Interactive(tree.body)
code = compile(tree, "<console internal>", "single", self.compile.compiler.flags, 1)
self.runcode(code)
def run_cell_simple(code, user_ns):
nodelist = ast.parse(code).body
nodes_exec, nodes_interactive = nodelist[:-1], nodelist[-1:]
bytecodes = []
for node in nodes_exec:
node = ast.Module([node])
bytecode = compile(node, '<string>', 'exec')
bytecodes.append(bytecode)
for node in nodes_interactive:
node = ast.Interactive([node])
bytecode = compile(node, '<string>', 'single')
bytecodes.append(bytecode)
for bytecode in bytecodes:
exec(bytecode, globals(), user_ns)
def generate(tree):
"""
>>> from pypethon.lexer import lex
>>> from pypethon.parser import parse
>>> from pypethon.stdlib import build
>>> state = build()
>>> exec(generate(parse(lex("|= plus5 inc | inc | inc | inc | inc"))), state)
>>> exec(generate(parse(lex("= ans 33 | inc | inc | inc | inc | plus5"))), state)
>>> exec(generate(parse(lex("ans"))), state)
42
"""
return compile(
source=ast.fix_missing_locations(
ast.Interactive(body=[wrap(translate(tree))])
),
filename="Pypethon",
mode="single",
)
