def visit(self, tree):
if isinstance(tree, HyExpression) and tree != []:
call = tree[0]
if call == "fn" and self.should_hoist():
new_name = HySymbol(self.unique_name())
new_name.replace(tree)
fn_def = HyExpression([HySymbol("def"),
new_name,
tree])
fn_def.replace(tree)
self.hoist(fn_def)
return new_name
def visit(self, tree):
"""
Visit all the nodes in the Hy code tree.
"""
if isinstance(tree, HyExpression) and tree != []:
call = tree[0]
if call == "fn" and self.should_hoist():
# if we have a Function and we should hoist it --
new_name = HySymbol(self.unique_name())
new_name.replace(tree)
fn_def = HyExpression([HySymbol("def"),
new_name,
tree])
fn_def.replace(tree)
self.hoist(fn_def)
return new_name
def compile_symbol(self, symbol):
if "." in symbol:
glob, local = symbol.rsplit(".", 1)
glob = HySymbol(glob)
glob.replace(symbol)
return ast.Attribute(
lineno=symbol.start_line,
col_offset=symbol.start_column,
value=self.compile_symbol(glob),
attr=str(local),
ctx=ast.Load()
)
return ast.Name(id=str(symbol), ctx=ast.Load(),
lineno=symbol.start_line,
col_offset=symbol.start_column)
def test_sets():
""" Ensure that we can tokenize a set. """
objs = tokenize("#{1 2}")
assert objs == [HySet([HyInteger(1), HyInteger(2)])]
objs = tokenize("(bar #{foo bar baz})")
assert objs == [
HyExpression([HySymbol("bar"),
HySet(["foo", "bar", "baz"])])
]
objs = tokenize("#{(foo bar) (baz quux)}")
assert objs == [
HySet([
HyExpression([HySymbol("foo"), HySymbol("bar")]),
HyExpression([HySymbol("baz"), HySymbol("quux")])
])
]
def compile_symbol(self, symbol):
if "." in symbol:
glob, local = symbol.rsplit(".", 1)
glob = HySymbol(glob)
glob.replace(symbol)
return ast.Attribute(lineno=symbol.start_line,
col_offset=symbol.start_column,
value=self.compile_symbol(glob),
attr=ast_str(local),
ctx=ast.Load())
return ast.Name(id=ast_str(symbol),
arg=ast_str(symbol),
ctx=ast.Load(),
lineno=symbol.start_line,
col_offset=symbol.start_column)
def take_macro(tree):
tree.pop(0) # "take"
n = tree.pop(0)
ret = tree.pop(0)
return HyExpression([HySymbol('slice'),
ret,
HyInteger(0),
HyInteger(n)])
def test_lex_fractions():
""" Make sure that fractions are valid expressions"""
objs = tokenize("1/2")
assert objs == [
HyExpression([HySymbol("fraction"),
HyInteger(1),
HyInteger(2)])
]
def test_dicts():
""" Ensure that we can tokenize a dict. """
objs = tokenize("{foo bar bar baz}")
assert objs == [HyDict(["foo", "bar", "bar", "baz"])]
objs = tokenize("(bar {foo bar bar baz})")
assert objs == [
HyExpression([HySymbol("bar"),
HyDict(["foo", "bar", "bar", "baz"])])
]
objs = tokenize("{(foo bar) (baz quux)}")
assert objs == [
HyDict([
HyExpression([HySymbol("foo"), HySymbol("bar")]),
HyExpression([HySymbol("baz"), HySymbol("quux")])
])
]
def let_macro(tree):
tree.pop(0) # "let"
variables = tree.pop(0)
# tree is now the body
expr = HyExpression([HySymbol("fn"), HyList([])])
for var in variables:
if isinstance(var, list):
expr.append(HyExpression([HySymbol("setf"),
var[0], var[1]]))
else:
expr.append(HyExpression([HySymbol("setf"),
var, HySymbol("None")]))
for stmt in tree:
expr.append(stmt)
return HyExpression([expr])
def t_identifier(p):
obj = p[0].value
try:
return HyInteger(obj)
except ValueError:
pass
if '/' in obj:
try:
lhs, rhs = obj.split('/')
return HyExpression(
[HySymbol('fraction'),
HyInteger(lhs),
HyInteger(rhs)])
except ValueError:
pass
try:
return HyFloat(obj)
except ValueError:
pass
if obj != 'j':
try:
return HyComplex(obj)
except ValueError:
pass
table = {
"true": "True",
"false": "False",
"nil": "None",
}
if obj in table:
return HySymbol(table[obj])
if obj.startswith(":"):
return HyKeyword(obj)
obj = ".".join([hy_symbol_mangle(part) for part in obj.split(".")])
return HySymbol(obj)
def compile_dotted_expression(self, expr):
ofn = expr.pop(0) # .join
fn = HySymbol(ofn[1:])
fn.replace(ofn)
obj = expr.pop(0) # [1 2 3 4]
return ast.Call(func=ast.Attribute(lineno=expr.start_line,
col_offset=expr.start_column,
value=self.compile(obj),
attr=ast_str(fn),
ctx=ast.Load()),
args=[self.compile(x) for x in expr],
keywords=[],
lineno=expr.start_line,
col_offset=expr.start_column,
starargs=None,
kwargs=None)
def hy_compile(tree, root=None):
" Compile a HyObject tree into a Python AST tree. "
compiler = HyASTCompiler()
tlo = root
if root is None:
tlo = ast.Module
_ast = compiler.compile(tree)
if type(_ast) == list:
_ast = compiler._mangle_branch(_ast, 0, 0)
if hasattr(sys, "subversion"):
implementation = sys.subversion[0].lower()
elif hasattr(sys, "implementation"):
implementation = sys.implementation.name.lower()
imports = []
for package in compiler.imports:
imported = set()
syms = compiler.imports[package]
for entry, form in syms:
if entry in imported:
continue
replace = form
if implementation != "cpython":
# using form causes pypy to blow up; let's conditionally
# add this for cpython, since it won't go through and make
# sure the AST makes sense. Muhahaha. - PRT
replace = tree[0]
imported.add(entry)
imports.append(
HyExpression([
HySymbol("import"),
HyList([HySymbol(package),
HyList([HySymbol(entry)])])
]).replace(replace))
_ast = compiler.compile(imports) + _ast
ret = tlo(body=_ast)
return ret
def visit(self, tree):
if isinstance(tree, HyExpression) and tree != []:
call = tree[0]
if call == "if" and self.should_hoist():
# If we've got a hoistable if statement
fn = HyExpression(
[HyExpression([HySymbol("fn"),
HyList([]), tree])])
fn.replace(tree)
return fn
def t_identifier(p):
obj = p[0].value
try:
return HyInteger(obj)
except ValueError:
pass
try:
return HyFloat(obj)
except ValueError:
pass
if obj != 'j':
try:
return HyComplex(obj)
except ValueError:
pass
table = {
"true": "True",
"false": "False",
"null": "None",
}
if obj in table:
return HySymbol(table[obj])
if obj.startswith(":"):
return HyKeyword(obj)
if obj.startswith("&"):
return HyLambdaListKeyword(obj)
if obj.startswith("*") and obj.endswith("*") and obj not in ("*", "**"):
obj = obj[1:-1].upper()
if "-" in obj and obj != "-":
obj = obj.replace("-", "_")
return HySymbol(obj)
def test_dicts():
""" Ensure that we can tokenize a dict. """
objs = tokenize("{foo bar bar baz}")
assert objs == [HyDict({
"foo": "bar",
"bar": "baz"
})]
objs = tokenize("(bar {foo bar bar baz})")
assert objs == [HyExpression([HySymbol("bar"),
HyDict({"foo": "bar",
"bar": "baz"})])]
def test_compiler_bare_names(self):
"""
Check that the compiler doesn't drop bare names from code branches
"""
ret = self.c.compile(
self._make_expression(HySymbol("do"), HySymbol("a"), HySymbol("b"),
HySymbol("c")))
# We expect two statements and a final expr.
self.assertEqual(len(ret.stmts), 2)
stmt = ret.stmts[0]
self.assertIsInstance(stmt, ast.Expr)
self.assertIsInstance(stmt.value, ast.Name)
self.assertEqual(stmt.value.id, "a")
stmt = ret.stmts[1]
self.assertIsInstance(stmt, ast.Expr)
self.assertIsInstance(stmt.value, ast.Name)
self.assertEqual(stmt.value.id, "b")
expr = ret.expr
self.assertIsInstance(expr, ast.Name)
self.assertEqual(expr.id, "c")
def compile_dotted_expression(self, expr):
ofn = expr.pop(0) # .join
fn = HySymbol(ofn[1:])
fn.replace(ofn)
obj = expr.pop(0) # [1 2 3 4]
return ast.Call(
func=ast.Attribute(
lineno=expr.start_line,
col_offset=expr.start_column,
value=self.compile(obj),
attr=ast_str(fn),
ctx=ast.Load()),
args=[self.compile(x) for x in expr],
keywords=[],
lineno=expr.start_line,
col_offset=expr.start_column,
starargs=None,
kwargs=None)
def test_sets():
""" Ensure that we can tokenize a set. """
objs = tokenize("#{1 2}")
assert objs == [HySet([HyInteger(1), HyInteger(2)])]
objs = tokenize("(bar #{foo bar baz})")
assert objs == [
HyExpression([HySymbol("bar"),
HySet(["foo", "bar", "baz"])])
]
objs = tokenize("#{(foo bar) (baz quux)}")
assert objs == [
HySet([
HyExpression([HySymbol("foo"), HySymbol("bar")]),
HyExpression([HySymbol("baz"), HySymbol("quux")])
])
]
# Duplicate items in a literal set should be okay (and should
# be preserved).
objs = tokenize("#{1 2 1 1 2 1}")
assert objs == [HySet([HyInteger(n) for n in [1, 2, 1, 1, 2, 1]])]
assert len(objs[0]) == 6
# https://github.com/hylang/hy/issues/1120
objs = tokenize("#{a 1}")
assert objs == [HySet([HySymbol("a"), HyInteger(1)])]
def test_preprocessor_expression():
""" Test that macro expansion doesn't recurse"""
obj = macroexpand(tokenize('(test (test "one" "two"))')[0], __name__)
assert type(obj) == HyList
assert type(obj[0]) == HyExpression
assert obj[0] == HyExpression([HySymbol("test"),
HyString("one"),
HyString("two")])
obj = HyList([HyString("one"), HyString("two")])
obj = tokenize('(shill ["one" "two"])')[0][1]
assert obj == macroexpand(obj, '')
def koan_macro():
return HyExpression([HySymbol('print'),
HyString("""
Ummon asked the head monk, "What sutra are you lecturing on?"
"The Nirvana Sutra."
"The Nirvana Sutra has the Four Virtues, hasn't it?"
"It has."
Ummon asked, picking up a cup, "How many virtues has this?"
"None at all," said the monk.
"But ancient people said it had, didn't they?" said Ummon.
"What do you think of what they said?"
Ummon struck the cup and asked, "You understand?"
"No," said the monk.
"Then," said Ummon, "You'd better go on with your lectures on the sutra."
""")])
def test_fn_compiler_empty_function(self):
ret = self.c.compile_function_def(
self._make_expression(HySymbol("fn"), HyList()))
self.assertEqual(ret.imports, {})
self.assertEqual(len(ret.stmts), 1)
stmt = ret.stmts[0]
self.assertIsInstance(stmt, ast.FunctionDef)
self.assertIsInstance(stmt.args, ast.arguments)
self.assertEqual(stmt.args.vararg, None)
self.assertEqual(stmt.args.kwarg, None)
self.assertEqual(stmt.args.defaults, [])
self.assertEqual(stmt.decorator_list, [])
self.assertEqual(len(stmt.body), 1)
self.assertIsInstance(stmt.body[0], ast.Pass)
self.assertIsInstance(ret.expr, ast.Name)
def test_lex_mangling_qmark():
"""Ensure that identifiers ending with a question mark get mangled ok"""
entry = tokenize("foo?")
assert entry == [HySymbol("is_foo")]
entry = tokenize("?")
assert entry == [HySymbol("?")]
entry = tokenize("im?foo")
assert entry == [HySymbol("im?foo")]
entry = tokenize(".foo?")
assert entry == [HySymbol(".is_foo")]
entry = tokenize("foo.bar?")
assert entry == [HySymbol("foo.is_bar")]
entry = tokenize("foo?.bar")
assert entry == [HySymbol("is_foo.bar")]
entry = tokenize(".foo?.bar.baz?")
assert entry == [HySymbol(".is_foo.bar.is_baz")]
def test_lex_mangling_bang():
"""Ensure that identifiers ending with a bang get mangled ok"""
entry = tokenize("foo!")
assert entry == [HySymbol("foo_bang")]
entry = tokenize("!")
assert entry == [HySymbol("!")]
entry = tokenize("im!foo")
assert entry == [HySymbol("im!foo")]
entry = tokenize(".foo!")
assert entry == [HySymbol(".foo_bang")]
entry = tokenize("foo.bar!")
assert entry == [HySymbol("foo.bar_bang")]
entry = tokenize("foo!.bar")
assert entry == [HySymbol("foo_bang.bar")]
entry = tokenize(".foo!.bar.baz!")
assert entry == [HySymbol(".foo_bang.bar.baz_bang")]
def router(tree, rkwargs=None):
tree = HyExpression(tree)
name = tree.pop(0)
path = tree.pop(0)
tree.insert(0, HySymbol("fn"))
tree = HyExpression([HySymbol("def"), name, tree])
route = HyExpression([HySymbol(".route"), HySymbol("app"), path])
if rkwargs:
route = HyExpression([
HySymbol("kwapply"), route,
HyDict({HyString("methods"): rkwargs})
])
return HyExpression([HySymbol("with_decorator"), route, tree])
def term_unquote_splice(p):
return HyExpression([HySymbol("unquote_splice"), p[1]])
def hash_reader(p):
st = p[0].getstr()[1]
str_object = HyString(st)
expr = p[1]
return HyExpression([HySymbol("dispatch_reader_macro"), str_object, expr])
def test_lex_symbols():
""" Make sure that symbols are valid expressions"""
objs = tokenize("foo ")
assert objs == [HySymbol("foo")]
def test_lex_expression_integer():
""" Make sure expressions can produce integers """
objs = tokenize("(foo 2)")
assert objs == [HyExpression([HySymbol("foo"), HyInteger(2)])]
def test_lex_expression_strings():
""" Test that expressions can produce symbols """
objs = tokenize("(foo \"bar\")")
assert objs == [HyExpression([HySymbol("foo"), HyString("bar")])]
def test_lex_expression_symbols():
""" Make sure that expressions produce symbols """
objs = tokenize("(foo bar)")
assert objs == [HyExpression([HySymbol("foo"), HySymbol("bar")])]
def test_simple_cons():
"""Check that cons gets tokenized correctly"""
entry = tokenize("(a . b)")[0]
assert entry == HyCons(HySymbol("a"), HySymbol("b"))
def test_lex_mangling_hyphen():
"""Ensure that hyphens get translated to underscores during mangling"""
entry = tokenize("foo-bar")
assert entry == [HySymbol("foo_bar")]
entry = tokenize("-")
assert entry == [HySymbol("-")]
def test_lex_comment_382():
"""Ensure that we can tokenize sources with a comment at the end"""
entry = tokenize("foo ;bar\n;baz")
assert entry == [HySymbol("foo")]
def test_reader_macro():
"""Ensure reader macros are handles properly"""
entry = tokenize("#^()")
assert entry[0][0] == HySymbol("dispatch_reader_macro")
assert entry[0][1] == HyString("^")
assert len(entry[0]) == 3
