def test_ast_bad_type(): "Make sure AST breakage can happen" class C: pass with pytest.raises(TypeError): hy_compile(C(), __name__, filename='<string>', source='')
def test_ast_bad_type(): "Make sure AST breakage can happen" try: hy_compile("foo", "__main__") assert True is False except HyCompileError: pass
def cant_compile(expr): expr = tokenize(expr) try: hy_compile(expr) assert False except HyCompileError: pass
def cant_compile(expr): with pytest.raises(HyError) as excinfo: hy_compile(hy_parse(expr), __name__) # Anything that can't be compiled should raise a user friendly # error, otherwise it's a compiler bug. assert issubclass(excinfo.type, HyLanguageError) assert excinfo.value.msg return excinfo.value
def test_ast_non_kwapplyable(): """ Ensure kwapply breaks """ code = tokenize("(kwapply foo bar)") code[0][2] = None try: hy_compile(code, "__main__") assert True is False except HyCompileError: pass
def test_compile_error(): """Ensure we get compile error in tricky cases""" try: hy_compile(tokenize("(fn [] (= 1))")) except HyCompileError as e: assert(str(e) == "`=' needs at least 2 arguments, got 1 (line 1, column 8)") else: assert(False)
def test_ast_bad_type(): "Make sure AST breakage can happen" class C: pass try: hy_compile(C(), "__main__") assert True is False except TypeError: pass
def cant_compile(expr): expr = tokenize(expr) try: hy_compile(expr, "__main__") assert False except HyCompileError as e: # Anything that can't be compiled should raise a user friendly # error, otherwise it's a compiler bug. assert isinstance(e.exception, HyTypeError) assert e.traceback
def cant_compile(expr): with pytest.raises(HyError) as excinfo: hy_compile(hy_parse(expr), __name__) if issubclass(excinfo.type, HyLanguageError): assert excinfo.value.msg return excinfo.value elif issubclass(excinfo.type, HyCompileError): # Anything that can't be compiled should raise a user friendly # error, otherwise it's a compiler bug. return excinfo.value
def test_unicode_exception_messages(): """Ensure we can generate error messages containing unicode.""" try: hy_compile(tokenize("(defmacro ❤ () (throw Exception)) (❤)"), "__main__") assert False except HyMacroExpansionError as e: assert isinstance(e.expression, HyObject) message = str_type(e) assert '❤' in message assert "Exception()" in message
def hy_eval(hytree, namespace, module_name): foo = HyObject() foo.start_line = 0 foo.end_line = 0 foo.start_column = 0 foo.end_column = 0 replace_hy_obj(hytree, foo) if not isinstance(module_name, string_types): raise HyTypeError(foo, "Module name must be a string") _ast, expr = hy_compile(hytree, module_name, get_expr=True) # Spoof the positions in the generated ast... for node in ast.walk(_ast): node.lineno = 1 node.col_offset = 1 for node in ast.walk(expr): node.lineno = 1 node.col_offset = 1 if not isinstance(namespace, dict): raise HyTypeError(foo, "Globals must be a dictionary") # Two-step eval: eval() the body of the exec call eval(ast_compile(_ast, "<eval_body>", "exec"), namespace) # Then eval the expression context and return that return eval(ast_compile(expr, "<eval>", "eval"), namespace)
def hylang(self, line, cell=None, filename='<input>', symbol='single'): """ Ipython magic function for running hylang code in ipython Use %hylang one line of code or %%hylang for a block or cell Note that we pass the AST directly to IPython.""" global SIMPLE_TRACEBACKS source = cell if cell else line try: tokens = tokenize(source) except PrematureEndOfInput: print( "Premature End of Input" ) except LexException as e: if e.source is None: e.source = source e.filename = filename print(str(e)) try: _ast = hy_compile(tokens, "__console__", root=ast.Interactive) self.shell.run_ast_nodes(_ast.body,'<input>',compiler=ast_compile) except HyTypeError as e: if e.source is None: e.source = source e.filename = filename if SIMPLE_TRACEBACKS: print(str(e)) else: self.shell.showtraceback() except Exception: self.shell.showtraceback()
def hylang(self, line, cell=None, filename='<input>', symbol='single'): """ Ipython magic function for running hylang code in ipython Use %hylang one line of code or %%hylang for a block or cell Note that we pass the AST directly to IPython.""" global SIMPLE_TRACEBACKS source = cell if cell else line try: tokens = tokenize(source) except PrematureEndOfInput: print("Premature End of Input") except LexException as e: if e.source is None: e.source = source e.filename = filename print(str(e)) try: _ast = hy_compile(tokens, "__console__", root=ast.Interactive) self.shell.run_ast_nodes(_ast.body, '<input>', compiler=ast_compile) except HyTypeError as e: if e.source is None: e.source = source e.filename = filename if SIMPLE_TRACEBACKS: print(str(e)) else: self.shell.showtraceback() except Exception: self.shell.showtraceback()
def runsource(self, source, filename='<input>', symbol='single'): global SIMPLE_TRACEBACKS try: tokens = tokenize(source) except PrematureEndOfInput: return True except LexException as e: if e.source is None: e.source = source e.filename = filename sys.stderr.write(str(e)) return False try: _ast = hy_compile(tokens, "__console__", root=ast.Interactive) if self.spy: print_python_code(_ast) code = ast_compile(_ast, filename, symbol) except HyTypeError as e: if e.source is None: e.source = source e.filename = filename if SIMPLE_TRACEBACKS: sys.stderr.write(str(e)) else: self.showtraceback() return False except Exception: self.showtraceback() return False self.runcode(code) return False
def on_command(self, command, args, chan, *_args, **_kwargs): if command == "!hy": if hy is None: self.reply(chan, "Hy is not installed") return try: tree = hy_compile(hy.read_str(args), "__main__") except Exception as ex: # I don't like catching all but I don't know how many errors this can throw self.reply(chan, "Parsing Error: " + str(ex)) return else: try: tree = ast.parse(args) except SyntaxError as ex: self.reply(chan, "Syntax Error: " + str(ex)) return returnval = None error = None self.aeval.writer = io.StringIO() try: returnval = self.aeval.eval_ast(tree) except asteval.UserError as e: error = e.error except asteval.EvalError as e: error = e except RecursionError as e: error = e if returnval is not None: self.reply(chan, str(returnval)) stdout = self.aeval.writer.getvalue() if stdout: self.reply(chan, stdout) if error is not None: self.reply(chan, str(error.__class__.__name__) + ": " + str(error))
def do_execute_direct(self, code): ''' Exceute the code, and return result. ''' self.result = None #### try to parse it: try: tokens = tokenize(code) _ast = hy_compile(tokens, '', root=ast.Interactive, compiler=self.compiler) code = compile(_ast, "In [%s]" % self.execution_count, mode="single") # calls sys.displayhook: eval(code, self.env) except Exception as e: self.Error(traceback.format_exc()) self.kernel_resp.update({ "status": "error", 'ename': e.__class__.__name__, # Exception name, as a string 'evalue': e.__class__.__name__, # Exception value, as a string 'traceback': [], # traceback frames as strings }) return None return self.result
def do_execute_direct(self, code): ''' Execute the code, and return result. ''' self.result = None #### try to parse it: try: filename = "In [%s]" % self.execution_count hy_ast = hy_parse(code, filename=filename) exec_ast, eval_ast = hy_compile(hy_ast, self.module, root=ast.Interactive, get_expr=True, compiler=self.hy_compiler, filename=filename, source=code) exec_code = compile(exec_ast, filename, 'single') eval_code = compile(eval_ast, filename, 'eval') eval(exec_code, self.locals) self.result = eval(eval_code, self.locals) except Exception as e: self.Error(traceback.format_exc()) self.kernel_resp.update({ "status": "error", 'ename' : e.__class__.__name__, # Exception name, as a string 'evalue' : e.__class__.__name__, # Exception value, as a string 'traceback' : [], # traceback frames as strings }) return None return self.result
def __compile_code(self, code_string): hytree = hy.lex.tokenize(code_string) module_name = '__main__' foo = HyObject() foo.start_line = 0 foo.end_line = 0 foo.start_column = 0 foo.end_column = 0 replace_hy_obj(hytree, foo) if not isinstance(module_name, string_types): raise HyTypeError(foo, "Module name must be a string") _ast, expr = hy_compile(hytree, module_name, get_expr=True) # Spoof the positions in the generated ast... for node in ast.walk(_ast): node.lineno = 1 node.col_offset = 1 for node in ast.walk(expr): node.lineno = 1 node.col_offset = 1 return _ast, expr
def runsource(self, source, filename='<input>', symbol='single'): global _machine try: _machine.process(source + "\n") except LexException: _machine = Machine(Idle, 1, 0) self.showsyntaxerror(filename) return False if type(_machine.state) != Idle: _machine = Machine(Idle, 1, 0) return True try: tokens = process(_machine.nodes, "__console__") except Exception: _machine = Machine(Idle, 1, 0) self.showtraceback() return False _machine = Machine(Idle, 1, 0) try: _ast = hy_compile(tokens, "__console__", root=ast.Interactive) code = ast_compile(_ast, filename, symbol) except Exception: self.showtraceback() return False self.runcode(code) return False
def runsource(self, source, filename="<input>", symbol="single"): global SIMPLE_TRACEBACKS try: tokens = tokenize(source) except PrematureEndOfInput: return True except LexException as e: if e.source is None: e.source = source e.filename = filename sys.stderr.write(str(e)) return False try: _ast = hy_compile(tokens, "__console__", root=ast.Interactive) if self.spy: print_python_code(_ast) code = ast_compile(_ast, filename, symbol) except HyTypeError as e: if e.source is None: e.source = source e.filename = filename if SIMPLE_TRACEBACKS: sys.stderr.write(str(e)) else: self.showtraceback() return False except Exception: self.showtraceback() return False self.runcode(code) return False
def test_ast_print(): code = hy_compile(tokenize("(print \"foo\")")).body[0] if sys.version_info[0] >= 3: assert type(code.value) == ast.Call return assert type(code) == ast.Print
def hy_eval(hytree, namespace, module_name, ast_callback=None): foo = HyObject() foo.start_line = 0 foo.end_line = 0 foo.start_column = 0 foo.end_column = 0 replace_hy_obj(hytree, foo) if not isinstance(module_name, string_types): raise HyTypeError(foo, "Module name must be a string") _ast, expr = hy_compile(hytree, module_name, get_expr=True) # Spoof the positions in the generated ast... for node in ast.walk(_ast): node.lineno = 1 node.col_offset = 1 for node in ast.walk(expr): node.lineno = 1 node.col_offset = 1 if ast_callback: ast_callback(_ast, expr) if not isinstance(namespace, dict): raise HyTypeError(foo, "Globals must be a dictionary") # Two-step eval: eval() the body of the exec call eval(ast_compile(_ast, "<eval_body>", "exec"), namespace) # Then eval the expression context and return that return eval(ast_compile(expr, "<eval>", "eval"), namespace)
def runsource(self, source, filename="<input>", symbol="single"): global _machine try: _machine.process(source + "\n") except LexException: _machine = Machine(Idle, 1, 0) self.showsyntaxerror(filename) return False if type(_machine.state) != Idle: _machine = Machine(Idle, 1, 0) return True try: tokens = process(_machine.nodes, "__console__") except Exception: _machine = Machine(Idle, 1, 0) self.showtraceback() return False _machine = Machine(Idle, 1, 0) try: _ast = hy_compile(tokens, "__console__", root=ast.Interactive) code = ast_compile(_ast, filename, symbol) except Exception: self.showtraceback() return False self.runcode(code) return False
def cant_compile(expr): try: hy_compile(import_buffer_to_hst(expr), "__main__") assert False except HyTypeError as e: # Anything that can't be compiled should raise a user friendly # error, otherwise it's a compiler bug. assert isinstance(e.expression, HyObject) assert e.message return e except HyCompileError as e: # Anything that can't be compiled should raise a user friendly # error, otherwise it's a compiler bug. assert isinstance(e.exception, HyTypeError) assert e.traceback return e
def _hy_source_to_code(self, data, path, _optimize=-1): if _could_be_hy_src(path): source = data.decode("utf-8") hy_tree = hy_parse(source, filename=path) with loader_module_obj(self) as module: data = hy_compile(hy_tree, module) return _py_source_to_code(self, data, path, _optimize=_optimize)
def getAST(code): try: a = tokenize(code) b = hy_compile(a, "hdb") ret = astor.dump(b) except PrematureEndOfInput: ret = "missing a paren" return ret
def getPython(code): try: a = tokenize(code) b = hy_compile(a, "hdb") ret = astor.codegen.to_source(b) except PrematureEndOfInput: ret = "missing a paren" return ret
def validate(self, code): try: tokens = tokenize(code.text) except PrematureEndOfInput: raise ValidationError(message='Unexpected end of input', index=len(code.text)) except LexException as e: raise ValidationError(message=str(e), index=len(code.text)) try: hy_compile(tokens, "__console__", root=ast.Interactive) except Exception as e: if hasattr(e, "message"): raise ValidationError(message='Syntax Error:' + e.message, index=len(code.text)) else: raise ValidationError(message='Syntax Error:' + str(e), index=len(code.text))
def _compile_string(s): hy_s = HyString(s) code = hy_compile([hy_s], "__main__") # We put hy_s in a list so it isn't interpreted as a docstring. # code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Str(s=xxx)]))]) return code.body[0].value.elts[0].s
def _compile_string(s): hy_s = HyString(s) code = hy_compile([hy_s], __name__, filename='<string>', source=s) # We put hy_s in a list so it isn't interpreted as a docstring. # code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Str(s=xxx)]))]) return code.body[0].value.elts[0].s
def hy_eval(hytree, namespace): foo = HyObject() foo.start_line = 0 foo.end_line = 0 foo.start_column = 0 foo.end_column = 0 hytree.replace(foo) _ast = hy_compile(hytree, root=ast.Expression) return eval(compile_(_ast, "<eval>", "eval"), namespace)
def _compile_string(s): hy_s = HyString(s) hy_s.start_line = hy_s.end_line = 0 hy_s.start_column = hy_s.end_column = 0 code = hy_compile([hy_s], "__main__") # code == ast.Module(body=[ast.Expr(value=ast.Str(s=xxx))]) return code.body[0].value.s
def hy_eval(hytree, namespace): foo = HyObject() foo.start_line = 0 foo.end_line = 0 foo.start_column = 0 foo.end_column = 0 hytree.replace(foo) _ast = hy_compile(hytree, root=ast.Expression) return eval(ast_compile(_ast, "<eval>", "eval"), namespace)
def getAST(): code = edit_object.edit_text try: a = tokenize(code) b = hy_compile(a, "hdb") ret = astor.dump(b) except PrematureEndOfInput: ret = "missing a paren" return ret
def __call__(self, source, filename="<input>", symbol="single"): if source == 'pass': # We need to return a no-op to signal that no more input is needed. return (compile(source, filename, symbol), ) * 2 hash_digest = hashlib.sha1(source.encode("utf-8").strip()).hexdigest() name = '{}-{}'.format(filename.strip('<>'), hash_digest) try: hy_ast = hy_parse(source, filename=name) except Exception: # Capture a traceback without the compiler/REPL frames. sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() self._update_exc_info() raise self._cache(source, name) try: hy_ast = hy_parse(source, filename=filename) root_ast = ast.Interactive if symbol == 'single' else ast.Module # Our compiler doesn't correspond to a real, fixed source file, so # we need to [re]set these. self.hy_compiler.filename = filename self.hy_compiler.source = source exec_ast, eval_ast = hy_compile(hy_ast, self.module, root=root_ast, get_expr=True, compiler=self.hy_compiler, filename=filename, source=source, import_stdlib=False) if self.ast_callback: self.ast_callback(exec_ast, eval_ast) exec_code = super(HyCompile, self).__call__(exec_ast, name, symbol) eval_code = super(HyCompile, self).__call__(eval_ast, name, 'eval') except HyLanguageError: # Hy will raise exceptions during compile-time that Python would # raise during run-time (e.g. import errors for `require`). In # order to work gracefully with the Python world, we convert such # Hy errors to code that purposefully reraises those exceptions in # the places where Python code expects them. sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() self._update_exc_info() exec_code = super(HyCompile, self).__call__( 'import hy._compat; hy._compat.reraise(' '_hy_last_type, _hy_last_value, _hy_last_traceback)', name, symbol) eval_code = super(HyCompile, self).__call__('None', name, 'eval') return exec_code, eval_code
def hy2py_main(): options = dict(prog="hy2py", usage="%(prog)s [options] [FILE]", formatter_class=argparse.RawDescriptionHelpFormatter) parser = argparse.ArgumentParser(**options) parser.add_argument("FILE", type=str, nargs='?', help="Input Hy code (use STDIN if \"-\" or " "not provided)") parser.add_argument("--with-source", "-s", action="store_true", help="Show the parsed source structure") parser.add_argument("--with-ast", "-a", action="store_true", help="Show the generated AST") parser.add_argument("--without-python", "-np", action="store_true", help=("Do not show the Python code generated " "from the AST")) options = parser.parse_args(sys.argv[1:]) if options.FILE is None or options.FILE == '-': sys.path.insert(0, "") filename = '<stdin>' source = sys.stdin.read() else: filename = options.FILE set_path(filename) with io.open(options.FILE, 'r', encoding='utf-8') as source_file: source = source_file.read() with filtered_hy_exceptions(): hst = hy_parse(source, filename=filename) if options.with_source: _print_for_windows(hst) print() print() with filtered_hy_exceptions(): _ast = hy_compile(hst, '__main__', filename=filename, source=source) if options.with_ast: _print_for_windows(ast.dump(_ast)) print() print() if not options.without_python: _print_for_windows(ast.unparse(_ast)) parser.exit(0)
def parse(tokens, source, filename, shell, interactive): try: _ast = hy_compile(tokens, "__console__", root=interactive) shell.run_ast_nodes(_ast.body, filename, compiler=ast_compile) except HyTypeError as e: if e.source is None: e.source = source e.filename = filename print(e) except Exception: shell.showtraceback()
def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): """``eval`` evaluates a quoted expression and returns the value. The optional second and third arguments specify the dictionary of globals to use and the module name. The globals dictionary defaults to ``(local)`` and the module name defaults to the name of the current module. => (eval '(print "Hello World")) "Hello World" If you want to evaluate a string, use ``read-str`` to convert it to a form first: => (eval (read-str "(+ 1 1)")) 2""" if namespace is None: frame = inspect.stack()[1][0] namespace = inspect.getargvalues(frame).locals if module_name is None: m = inspect.getmodule(inspect.stack()[1][0]) module_name = '__eval__' if m is None else m.__name__ foo = HyObject() foo.start_line = 0 foo.end_line = 0 foo.start_column = 0 foo.end_column = 0 replace_hy_obj(hytree, foo) if not isinstance(module_name, string_types): raise HyTypeError(foo, "Module name must be a string") _ast, expr = hy_compile(hytree, module_name, get_expr=True) # Spoof the positions in the generated ast... for node in ast.walk(_ast): node.lineno = 1 node.col_offset = 1 for node in ast.walk(expr): node.lineno = 1 node.col_offset = 1 if ast_callback: ast_callback(_ast, expr) if not isinstance(namespace, dict): raise HyTypeError(foo, "Globals must be a dictionary") # Two-step eval: eval() the body of the exec call eval(ast_compile(_ast, "<eval_body>", "exec"), namespace) # Then eval the expression context and return that return eval(ast_compile(expr, "<eval>", "eval"), namespace)
def _compile_string(s): hy_s = hy.models.String(s) code = hy_compile([hy_s], __name__, filename="<string>", source=s, import_stdlib=False) # We put hy_s in a list so it isn't interpreted as a docstring. # code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Str(s=xxx)]))]) return code.body[0].value.elts[0].s
def __call__(self, source, filename="<input>", symbol="single"): if source == 'pass': # We need to return a no-op to signal that no more input is needed. return (compile(source, filename, symbol),) * 2 hash_digest = hashlib.sha1(source.encode("utf-8").strip()).hexdigest() name = '{}-{}'.format(filename.strip('<>'), hash_digest) try: hy_ast = hy_parse(source, filename=name) except Exception: # Capture a traceback without the compiler/REPL frames. sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() self._update_exc_info() raise self._cache(source, name) try: hy_ast = hy_parse(source, filename=filename) root_ast = ast.Interactive if symbol == 'single' else ast.Module # Our compiler doesn't correspond to a real, fixed source file, so # we need to [re]set these. self.hy_compiler.filename = filename self.hy_compiler.source = source exec_ast, eval_ast = hy_compile(hy_ast, self.module, root=root_ast, get_expr=True, compiler=self.hy_compiler, filename=filename, source=source) if self.ast_callback: self.ast_callback(exec_ast, eval_ast) exec_code = super(HyCompile, self).__call__(exec_ast, name, symbol) eval_code = super(HyCompile, self).__call__(eval_ast, name, 'eval') except HyLanguageError: # Hy will raise exceptions during compile-time that Python would # raise during run-time (e.g. import errors for `require`). In # order to work gracefully with the Python world, we convert such # Hy errors to code that purposefully reraises those exceptions in # the places where Python code expects them. sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() self._update_exc_info() exec_code = super(HyCompile, self).__call__( 'import hy._compat; hy._compat.reraise(' '_hy_last_type, _hy_last_value, _hy_last_traceback)', name, symbol) eval_code = super(HyCompile, self).__call__('None', name, 'eval') return exec_code, eval_code
def _hy_source_to_code(self, data, path, _optimize=-1): if _could_be_hy_src(path): source = data.decode("utf-8") try: hy_tree = hy_parse(source) data = hy_compile(hy_tree, self.name) except (HyTypeError, LexException) as e: if e.source is None: e.source = source e.filename = path raise return _py_source_to_code(self, data, path, _optimize=_optimize)
def test_ast_expression_basics(): """ Ensure basic AST expression conversion works. """ code = hy_compile(tokenize("(foo bar)")).body[0] tree = ast.Expr( value=ast.Call( func=ast.Name(id="foo", ctx=ast.Load()), args=[ast.Name(id="bar", ctx=ast.Load())], keywords=[], starargs=None, kwargs=None, ) ) _ast_spotcheck("value.func.id", code, tree)
def runsource(self, source, filename='<input>', symbol='single'): try: tokens = tokenize(source) except PrematureEndOfInput: return True except LexException: self.showsyntaxerror(filename) return False try: _ast = hy_compile(tokens, "__console__", root=ast.Interactive) code = ast_compile(_ast, filename, symbol) except Exception: self.showtraceback() return False self.runcode(code) return False
def byte_compile_hy(self, fullname=None): fullname = self._fix_name(fullname) if fullname is None: fullname = self.fullname hy_source = self.get_source(fullname) hy_tree = hy_parse(hy_source, filename=self.filename) with loader_module_obj(self) as module: hy_ast = hy_compile(hy_tree, module) code = compile(hy_ast, self.filename, 'exec', hy_ast_compile_flags) if not sys.dont_write_bytecode: try: hyc_compile(code, module=fullname) except IOError: pass return code
def hy_eval(hytree, namespace, module_name): foo = HyObject() foo.start_line = 0 foo.end_line = 0 foo.start_column = 0 foo.end_column = 0 hytree.replace(foo) _ast, expr = hy_compile(hytree, module_name, get_expr=True) # Spoof the positions in the generated ast... for node in ast.walk(_ast): node.lineno = 1 node.col_offset = 1 for node in ast.walk(expr): node.lineno = 1 node.col_offset = 1 # Two-step eval: eval() the body of the exec call eval(ast_compile(_ast, "<eval_body>", "exec"), namespace) # Then eval the expression context and return that return eval(ast_compile(expr, "<eval>", "eval"), namespace)