def test_escapes():
""" Ensure we can escape things """
entry = tokenize(r"""(foo "foo\n")""")[0]
assert entry[1] == "foo\n"
entry = tokenize(r"""(foo r"foo\s")""")[0]
assert entry[1] == r"foo\s"
def test_escapes():
""" Ensure we can escape things """
entry = tokenize("(foo \"foo\\n\")")[0]
assert entry[1] == "foo\n"
entry = tokenize("(foo \"foo\s\")")[0]
assert entry[1] == "foo\\s"
def test_complex():
"""Ensure we tokenize complex numbers properly"""
# This is a regression test for #143
entry = tokenize("(1j)")[0][0]
assert entry == HyComplex("1.0j")
entry = tokenize("(j)")[0][0]
assert entry == HySymbol("j")
def test_lex_expression_complex():
""" Make sure expressions can produce complex """
objs = tokenize("(foo 2.j)")
assert objs == [HyExpression([HySymbol("foo"), HyComplex(2.j)])]
objs = tokenize("(foo -0.5j)")
assert objs == [HyExpression([HySymbol("foo"), HyComplex(-0.5j)])]
objs = tokenize("(foo 1.e7j)")
assert objs == [HyExpression([HySymbol("foo"), HyComplex(1.e7j)])]
def test_lex_expression_float():
""" Make sure expressions can produce floats """
objs = tokenize("(foo 2.)")
assert objs == [HyExpression([HySymbol("foo"), HyFloat(2.)])]
objs = tokenize("(foo -0.5)")
assert objs == [HyExpression([HySymbol("foo"), HyFloat(-0.5)])]
objs = tokenize("(foo 1.e7)")
assert objs == [HyExpression([HySymbol("foo"), HyFloat(1.e7)])]
def test_lex_mangling_star():
"""Ensure that mangling starred identifiers works according to plan"""
entry = tokenize("*foo*")
assert entry == [HySymbol("FOO")]
entry = tokenize("*")
assert entry == [HySymbol("*")]
entry = tokenize("*foo")
assert entry == [HySymbol("*foo")]
def test_lex_single_quote_err():
"Ensure tokenizing \"' \" throws a LexException that can be stringified"
# https://github.com/hylang/hy/issues/1252
try:
tokenize("' ")
except LexException as e:
assert "Could not identify the next token" in str(e)
else:
assert False
def test_lex_bracket_strings():
objs = tokenize("#[my delim[hello world]my delim]")
assert objs == [HyString("hello world")]
assert objs[0].brackets == "my delim"
objs = tokenize("#[[squid]]")
assert objs == [HyString("squid")]
assert objs[0].brackets == ""
def test_lex_strings():
""" Make sure that strings are valid expressions"""
objs = tokenize('"foo"')
assert objs == [HyString("foo")]
# Make sure backslash-escaped newlines work (see issue #831)
objs = tokenize(r"""
"a\
bc"
""")
assert objs == [HyString("abc")]
def test_escapes():
""" Ensure we can escape things """
entry = tokenize("(foo \"foo\\n\")")[0]
assert entry[1] == "foo\n"
try:
entry = tokenize("(foo \"foo\s\")")[0]
assert True is False
except LexException:
pass
def test_cons_list():
"""Check that cons of something and a list gets tokenized as a list"""
entry = tokenize("(a . [])")[0]
assert entry == HyList([HySymbol("a")])
assert type(entry) == HyList
entry = tokenize("(a . ())")[0]
assert entry == HyExpression([HySymbol("a")])
assert type(entry) == HyExpression
entry = tokenize("(a b . {})")[0]
assert entry == HyDict([HySymbol("a"), HySymbol("b")])
assert type(entry) == HyDict
def test_preprocessor_expression():
""" Test inner macro expantion """
obj = process(tokenize('(test (test "one" "two"))')[0])
assert type(obj) == HyList
assert type(obj[0]) == HyList
assert obj[0] == HyList([HyString("one"), HyString("two")])
obj = HyList([HyString("one"), HyString("two")])
obj = tokenize('(shill ["one" "two"])')[0][1]
assert obj == process(obj)
def test_preprocessor_expression():
""" Test that macro expansion doesn't recurse"""
obj = macroexpand(tokenize('(test (test "one" "two"))')[0], HyASTCompiler(__name__))
assert type(obj) == HyList
assert type(obj[0]) == HyExpression
assert obj[0] == HyExpression([HySymbol("test"), HyString("one"), HyString("two")])
obj = HyList([HyString("one"), HyString("two")])
obj = tokenize('(shill ["one" "two"])')[0][1]
assert obj == macroexpand(obj, HyASTCompiler(""))
def test_dicts():
""" Ensure that we can tokenize a dict. """
objs = tokenize("{foo bar bar baz}")
assert objs == [HyDict({
"foo": "bar",
"bar": "baz"
})]
objs = tokenize("(bar {foo bar bar baz})")
assert objs == [HyExpression([HySymbol("bar"),
HyDict({"foo": "bar",
"bar": "baz"})])]
def test_sets():
""" Ensure that we can tokenize a set. """
objs = tokenize("#{1 2}")
assert objs == [HySet([HyInteger(1), HyInteger(2)])]
objs = tokenize("(bar #{foo bar baz})")
assert objs == [HyExpression([HySymbol("bar"),
HySet(["foo", "bar", "baz"])])]
objs = tokenize("#{(foo bar) (baz quux)}")
assert objs == [HySet([
HyExpression([HySymbol("foo"), HySymbol("bar")]),
HyExpression([HySymbol("baz"), HySymbol("quux")])
])]
def test_lex_exception():
""" Ensure tokenize throws a fit on a partial input """
try:
tokenize("(foo")
assert True is False
except LexException:
pass
try:
tokenize("&foo&")
assert True is False
except LexException:
pass
def test_unbalanced_exception():
"""Ensure the tokenization fails on unbalanced expressions"""
try:
tokenize("(bar))")
assert True is False
except LexException:
pass
try:
tokenize("(baz [quux]])")
assert True is False
except LexException:
pass
def test_lex_nan_and_inf():
assert isnan(tokenize("NaN")[0])
assert tokenize("Nan") == [HySymbol("Nan")]
assert tokenize("nan") == [HySymbol("nan")]
assert tokenize("NAN") == [HySymbol("NAN")]
assert tokenize("Inf") == [HyFloat(float("inf"))]
assert tokenize("inf") == [HySymbol("inf")]
assert tokenize("INF") == [HySymbol("INF")]
assert tokenize("-Inf") == [HyFloat(float("-inf"))]
assert tokenize("-inf") == [HySymbol("-inf")]
assert tokenize("-INF") == [HySymbol("-INF")]
def test_ast_good_try():
"Make sure AST can compile valid try"
hy_compile(tokenize("(try)"))
hy_compile(tokenize("(try 1)"))
hy_compile(tokenize("(try 1 (except) (else 1))"))
hy_compile(tokenize("(try 1 (else 1) (except))"))
hy_compile(tokenize("(try 1 (finally 1) (except))"))
hy_compile(tokenize("(try 1 (finally 1))"))
hy_compile(tokenize("(try 1 (except) (finally 1))"))
hy_compile(tokenize("(try 1 (except) (finally 1) (else 1))"))
hy_compile(tokenize("(try 1 (except) (else 1) (finally 1))"))
def test_dicts():
""" Ensure that we can tokenize a dict. """
objs = tokenize("{foo bar bar baz}")
assert objs == [HyDict(["foo", "bar", "bar", "baz"])]
objs = tokenize("(bar {foo bar bar baz})")
assert objs == [HyExpression([HySymbol("bar"),
HyDict(["foo", "bar",
"bar", "baz"])])]
objs = tokenize("{(foo bar) (baz quux)}")
assert objs == [HyDict([
HyExpression([HySymbol("foo"), HySymbol("bar")]),
HyExpression([HySymbol("baz"), HySymbol("quux")])
])]
def test_unicode_escapes():
"""Ensure unicode escapes are handled correctly"""
s = r'"a\xac\u1234\u20ac\U00008000"'
assert len(s) == 29
entry = tokenize(s)[0]
assert len(entry) == 5
assert [ord(x) for x in entry] == [97, 172, 4660, 8364, 32768]
def test_ast_print():
code = hy_compile(tokenize("(print \"foo\")")).body[0]
if sys.version_info[0] >= 3:
assert type(code.value) == ast.Call
return
assert type(code) == ast.Print
def eval(self, code):
"""Evals the given code.
Returns a dict with reponses."""
self.ret = []
# We might catch errors when tokenizing code.
try:
tokens = tokenize(code)
except:
self._format_excp(sys.exc_info())
return self.ret
# Setting stdout in a way so we can catch anything printed and return that
oldout = sys.stdout
oldin = sys.stdin
stdout = None
sys.stdout = StringIO()
for i in tokens:
try:
p = str(hy_eval(i, self.mod.__dict__, "__main__"))
except:
self._format_excp(sys.exc_info())
else:
self.ret.append({"value": p, "ns": 'None'})
# If there is nothing in return, we see if anything is in stdout
if p == "None":
stdout = sys.stdout.getvalue()
if stdout:
self.ret.append({'out': stdout})
sys.stdout = oldout
return self.ret
def test_preprocessor_simple():
""" Test basic macro expansion """
obj = macroexpand(tokenize('(test "one" "two")')[0],
__name__,
HyASTCompiler(__name__))
assert obj == HyList(["one", "two"])
assert type(obj) == HyList
def cant_compile(expr):
expr = tokenize(expr)
try:
hy_compile(expr)
assert False
except HyCompileError:
pass
def hylang(self, line, cell=None, filename='<input>', symbol='single'):
""" Ipython magic function for running hylang code in ipython
Use %hylang one line of code or
%%hylang for a block or cell
Note that we pass the AST directly to IPython."""
global SIMPLE_TRACEBACKS
source = cell if cell else line
try:
tokens = tokenize(source)
except PrematureEndOfInput:
print( "Premature End of Input" )
except LexException as e:
if e.source is None:
e.source = source
e.filename = filename
print(str(e))
try:
_ast = hy_compile(tokens, "__console__", root=ast.Interactive)
self.shell.run_ast_nodes(_ast.body,'<input>',compiler=ast_compile)
except HyTypeError as e:
if e.source is None:
e.source = source
e.filename = filename
if SIMPLE_TRACEBACKS:
print(str(e))
else:
self.shell.showtraceback()
except Exception:
self.shell.showtraceback()
def test_tag_macro_error():
"""Check if we get correct error with wrong dispatch character"""
try:
macroexpand(tokenize("(dispatch_tag_macro '- '())")[0],
HyASTCompiler(__name__))
except HyTypeError as e:
assert "with the character `-`" in str(e)
def runsource(self, source, filename="<input>", symbol="single"):
global SIMPLE_TRACEBACKS
try:
tokens = tokenize(source)
except PrematureEndOfInput:
return True
except LexException as e:
if e.source is None:
e.source = source
e.filename = filename
sys.stderr.write(str(e))
return False
try:
_ast = hy_compile(tokens, "__console__", root=ast.Interactive)
if self.spy:
print_python_code(_ast)
code = ast_compile(_ast, filename, symbol)
except HyTypeError as e:
if e.source is None:
e.source = source
e.filename = filename
if SIMPLE_TRACEBACKS:
sys.stderr.write(str(e))
else:
self.showtraceback()
return False
except Exception:
self.showtraceback()
return False
self.runcode(code)
return False
def test_dotted_list():
"""Check that dotted lists get tokenized correctly"""
entry = tokenize("(a b c . (d . e))")[0]
assert entry == HyCons(HySymbol("a"),
HyCons(HySymbol("b"),
HyCons(HySymbol("c"),
HyCons(HySymbol("d"),
HySymbol("e")))))
def test_preprocessor_exceptions():
""" Test that macro expansion raises appropriate exceptions"""
try:
macroexpand(tokenize('(defn)')[0], __name__)
assert False
except HyMacroExpansionError as e:
assert "_hy_anon_fn_" not in str(e)
assert "TypeError" not in str(e)
def test_preprocessor_simple():
"""Test basic macro expansion"""
obj = macroexpand(
tokenize('(test "one" "two")')[0], __name__, HyASTCompiler(__name__))
assert obj == List([String("one"), String("two")])
assert type(obj) == List
def import_buffer_to_hst(buf):
"""Import content from buf and return an Hy AST."""
return tokenize(buf + "\n")
def t(x): return tokenize("(foo {})".format(x))
def f(x): return [HyExpression([HySymbol("foo"), x])]
def test_lex_expression_integer():
""" Make sure expressions can produce integers """
objs = tokenize("(foo 2)")
assert objs == [HyExpression([HySymbol("foo"), HyInteger(2)])]
def test_lex_expression_symbols():
""" Make sure that expressions produce symbols """
objs = tokenize("(foo bar)")
assert objs == [HyExpression([HySymbol("foo"), HySymbol("bar")])]
def test_lex_mangling_hyphen():
"""Ensure that hyphens get translated to underscores during mangling"""
entry = tokenize("foo-bar")
assert entry == [HySymbol("foo_bar")]
entry = tokenize("-")
assert entry == [HySymbol("-")]
def test_reader_macro():
"""Ensure reader macros are handles properly"""
entry = tokenize("#^()")
assert entry[0][0] == HySymbol("dispatch_reader_macro")
assert entry[0][1] == HyString("^")
assert len(entry[0]) == 3
def can_compile(expr):
return hy_compile(tokenize(expr), "__main__")
def test_lex_big_float():
# https://github.com/hylang/hy/issues/1448
assert tokenize("1e900") == [HyFloat(1e900)]
assert tokenize("1e900-1e900j") == [HyComplex(1e900, -1e900)]
def test_lex_strings():
""" Make sure that strings are valid expressions"""
objs = tokenize("\"foo\" ")
assert objs == [HyString("foo")]
def f(x):
return hy_compile(tokenize(x), "__main__").body[0].value.s
def contains_import_from(code):
return any([
isinstance(node, ast.ImportFrom)
for node in hy_compile(tokenize(code), "__main__").body
])
def test_tag_macro():
"""Ensure tag macros are handled properly"""
entry = tokenize("#^()")
assert entry[0][0] == HySymbol("dispatch-tag-macro")
assert entry[0][1] == HyString("^")
assert len(entry[0]) == 3
def test_lex_bad_attrs():
with lexe():
tokenize("1.foo")
with lexe():
tokenize("0.foo")
with lexe():
tokenize("1.5.foo")
with lexe():
tokenize("1e3.foo")
with lexe():
tokenize("5j.foo")
with lexe():
tokenize("3+5j.foo")
with lexe():
tokenize("3.1+5.1j.foo")
assert tokenize("j.foo")
with lexe():
tokenize("3/4.foo")
assert tokenize("a/1.foo")
assert tokenize("1/a.foo")
with lexe():
tokenize(":hello.foo")
def test_hashbang():
""" Ensure we can escape things """
entry = tokenize("#!this is a comment\n")
assert entry == []
def test_unbalanced_exception():
"""Ensure the tokenization fails on unbalanced expressions"""
with lexe():
tokenize("(bar))")
with lexe():
tokenize("(baz [quux]])")
def test_lex_comment_382():
"""Ensure that we can tokenize sources with a comment at the end"""
entry = tokenize("foo ;bar\n;baz")
assert entry == [HySymbol("foo")]
def test_lex_single_quote_err():
"Ensure tokenizing \"' \" throws a LexException that can be stringified"
# https://github.com/hylang/hy/issues/1252
with lexe() as e:
tokenize("' ")
assert "Could not identify the next token" in str(e.value)
def test_simple_cons():
"""Check that cons gets tokenized correctly"""
entry = tokenize("(a . b)")[0]
assert entry == HyCons(HySymbol("a"), HySymbol("b"))
def test_discard():
"""Check that discarded terms are removed properly."""
# empty
assert tokenize("") == []
# single
assert tokenize("#_1") == []
# multiple
assert tokenize("#_1 #_2") == []
assert tokenize("#_1 #_2 #_3") == []
# nested discard
assert tokenize("#_ #_1 2") == []
assert tokenize("#_ #_ #_1 2 3") == []
# trailing
assert tokenize("0") == [0]
assert tokenize("0 #_1") == [0]
assert tokenize("0 #_1 #_2") == [0]
# leading
assert tokenize("2") == [2]
assert tokenize("#_1 2") == [2]
assert tokenize("#_0 #_1 2") == [2]
assert tokenize("#_ #_0 1 2") == [2]
# both
assert tokenize("#_1 2 #_3") == [2]
assert tokenize("#_0 #_1 2 #_ #_3 4") == [2]
# inside
assert tokenize("0 #_1 2") == [0, 2]
assert tokenize("0 #_1 #_2 3") == [0, 3]
assert tokenize("0 #_ #_1 2 3") == [0, 3]
# in HyList
assert tokenize("[]") == [HyList([])]
assert tokenize("[#_1]") == [HyList([])]
assert tokenize("[#_1 #_2]") == [HyList([])]
assert tokenize("[#_ #_1 2]") == [HyList([])]
assert tokenize("[0]") == [HyList([HyInteger(0)])]
assert tokenize("[0 #_1]") == [HyList([HyInteger(0)])]
assert tokenize("[0 #_1 #_2]") == [HyList([HyInteger(0)])]
assert tokenize("[2]") == [HyList([HyInteger(2)])]
assert tokenize("[#_1 2]") == [HyList([HyInteger(2)])]
assert tokenize("[#_0 #_1 2]") == [HyList([HyInteger(2)])]
assert tokenize("[#_ #_0 1 2]") == [HyList([HyInteger(2)])]
# in HySet
assert tokenize("#{}") == [HySet()]
assert tokenize("#{#_1}") == [HySet()]
assert tokenize("#{0 #_1}") == [HySet([HyInteger(0)])]
assert tokenize("#{#_1 0}") == [HySet([HyInteger(0)])]
# in HyDict
assert tokenize("{}") == [HyDict()]
assert tokenize("{#_1}") == [HyDict()]
assert tokenize("{#_0 1 2}") == [HyDict([HyInteger(1), HyInteger(2)])]
assert tokenize("{1 #_0 2}") == [HyDict([HyInteger(1), HyInteger(2)])]
assert tokenize("{1 2 #_0}") == [HyDict([HyInteger(1), HyInteger(2)])]
# in HyExpression
assert tokenize("()") == [HyExpression()]
assert tokenize("(#_foo)") == [HyExpression()]
assert tokenize("(#_foo bar)") == [HyExpression([HySymbol("bar")])]
assert tokenize("(foo #_bar)") == [HyExpression([HySymbol("foo")])]
assert tokenize("(foo :bar 1)") == [
HyExpression([HySymbol("foo"),
HyKeyword(":bar"),
HyInteger(1)])
]
assert tokenize("(foo #_:bar 1)") == [
HyExpression([HySymbol("foo"), HyInteger(1)])
]
assert tokenize("(foo :bar #_1)") == [
HyExpression([HySymbol("foo"), HyKeyword(":bar")])
]
# discard term with nesting
assert tokenize("[1 2 #_[a b c [d e [f g] h]] 3 4]") == [
HyList([HyInteger(1),
HyInteger(2),
HyInteger(3),
HyInteger(4)])
]
# discard with other prefix syntax
assert tokenize("a #_'b c") == [HySymbol("a"), HySymbol("c")]
assert tokenize("a '#_b c") == [
HySymbol("a"),
HyExpression([HySymbol("quote"), HySymbol("c")])
]
assert tokenize("a '#_b #_c d") == [
HySymbol("a"),
HyExpression([HySymbol("quote"), HySymbol("d")])
]
assert tokenize("a '#_ #_b c d") == [
HySymbol("a"),
HyExpression([HySymbol("quote"), HySymbol("d")])
]
def test_lex_expression_strings():
""" Test that expressions can produce strings """
objs = tokenize("(foo \"bar\")")
assert objs == [HyExpression([HySymbol("foo"), HyString("bar")])]
def test_lex_symbols():
""" Make sure that symbols are valid expressions"""
objs = tokenize("foo ")
assert objs == [HySymbol("foo")]
def test_lex_bad_attrs():
with lexe() as execinfo:
tokenize("1.foo")
check_ex(execinfo, [
' File "<string>", line 1\n',
' 1.foo\n',
' ^\n',
'LexException: Cannot access attribute on anything other'
' than a name (in order to get attributes of expressions,'
' use `(. <expression> <attr>)` or `(.<attr> <expression>)`)\n'])
with lexe(): tokenize("0.foo")
with lexe(): tokenize("1.5.foo")
with lexe(): tokenize("1e3.foo")
with lexe(): tokenize("5j.foo")
with lexe(): tokenize("3+5j.foo")
with lexe(): tokenize("3.1+5.1j.foo")
assert tokenize("j.foo")
with lexe(): tokenize("3/4.foo")
assert tokenize("a/1.foo")
assert tokenize("1/a.foo")
with lexe(): tokenize(":hello.foo")
def test_preprocessor_simple():
""" Test basic macro expansion """
obj = macroexpand(tokenize('(test "one" "two")')[0], __name__)
assert obj == HyList(["one", "two"])
assert type(obj) == HyList
def test_lex_fractions():
""" Make sure that fractions are valid expressions"""
objs = tokenize("1/2")
assert objs == [HyExpression([HySymbol("fraction"), HyInteger(1),
HyInteger(2)])]
def test_lex_integers():
""" Make sure that integers are valid expressions"""
objs = tokenize("42 ")
assert objs == [HyInteger(42)]
def test_preprocessor_exceptions():
"""Test that macro expansion raises appropriate exceptions"""
with pytest.raises(HyMacroExpansionError) as excinfo:
macroexpand(tokenize("(when)")[0], __name__, HyASTCompiler(__name__))
assert "_hy_anon_" not in excinfo.value.msg
def test_lex_digit_separators():
assert tokenize("1_000_000") == [HyInteger(1000000)]
assert tokenize("1,000,000") == [HyInteger(1000000)]
assert tokenize("1,000_000") == [HyInteger(1000000)]
assert tokenize("1_000,000") == [HyInteger(1000000)]
assert tokenize("0x_af") == [HyInteger(0xaf)]
assert tokenize("0x,af") == [HyInteger(0xaf)]
assert tokenize("0b_010") == [HyInteger(0b010)]
assert tokenize("0b,010") == [HyInteger(0b010)]
assert tokenize("0o_373") == [HyInteger(0o373)]
assert tokenize("0o,373") == [HyInteger(0o373)]
assert tokenize('1_2.3,4') == [HyFloat(12.34)]
assert tokenize('1_2e3,4') == [HyFloat(12e34)]
assert (tokenize("1,2/3_4") == [
HyExpression([HySymbol("fraction"),
HyInteger(12),
HyInteger(34)])
])
assert tokenize("1,0_00j") == [HyComplex(1000j)]
assert tokenize(",,,,___,__1__,,__,,2__,,,__") == [HyInteger(12)]
assert (tokenize(",,,,___,__1__,,__,,2__,q,__") == [
HySymbol(",,,,___,__1__,,__,,2__,q,__")
])
def t(x):
return tokenize("(foo {})".format(x))
def test_lex_exception():
""" Ensure tokenize throws a fit on a partial input """
with peoi(): tokenize("(foo")
with peoi(): tokenize("{foo bar")
with peoi(): tokenize("(defn foo [bar]")
with peoi(): tokenize("(foo \"bar")
