def test_braces(self):
self.assertEqual(lex('[a, 1]'),
[Term(Tag('['), None, None, None),
Term(Tag('IDENTIFIER'), 'a', None, None),
Term(Tag(','), None, None, None),
Term(Tag('.int.'), 1, None, None),
Term(Tag(']'), None, None, None)])
def test_dedent(self):
self.assertEqual(lex(SIMPLE_DEDENT),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag(':'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "blee", None, None),
Term(Tag('EOL'), None, None, None)])
def test_holes(self):
self.assertEqual(lex('${'), [Term(Tag('${'), None, None, None)])
self.assertEqual(lex('$blee'), [Term(Tag('DOLLAR_IDENT'), "blee", None, None)])
self.assertEqual(lex('@{'), [Term(Tag('@{'), None, None, None)])
self.assertEqual(lex('@fred'), [Term(Tag('AT_IDENT'), "fred", None, None)])
self.assertEqual(lex('@_fred'), [Term(Tag('AT_IDENT'), "_fred", None, None)])
self.assertEqual(lex('@_'), [Term(Tag('AT_IDENT'), "_", None, None)])
self.assertEqual(lex('$0'), [Term(Tag('$'), None, None, None),
Term(Tag(".int."), 0, None, None)])
self.assertEqual(lex('@1'), [Term(Tag('@'), None, None, None),
Term(Tag(".int."), 1, None, None)])
def test_arrow(self):
self.assertEqual(lex(ARROW_INDENT),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag('->'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('EOL'), None, None, None)])
def test_zap(self):
self.assertEqual(lex('1 >'), [Term(Tag('.int.'), 1, None, None),
Term(Tag('>'), None, None, None)])
self.assertEqual(lex('1 >='), [Term(Tag('.int.'), 1, None, None),
Term(Tag('>='), None, None, None)])
self.assertEqual(lex('1 >>='), [Term(Tag('.int.'), 1, None, None),
Term(Tag('>>='), None, None, None)])
def test_simpleTerm(self):
"""
Kernel syntax for terms is parsed properly.
"""
parse = self.getParser("baseTerm")
self.assertEqual(parse("x"), Term(Tag("x"), None, None))
self.assertEqual(parse("x()"), Term(Tag("x"), None, []))
self.assertEqual(parse("x(1)"),
Term(Tag("x"), None, (Term(Tag(".int."), 1, None), )))
self.assertEqual(
parse("x(1, 2)"),
Term(Tag("x"), None,
(Term(Tag(".int."), 1, None), Term(Tag(".int."), 2, None))))
self.assertEqual(parse("1"), Term(Tag(".int."), 1, None))
self.assertEqual(parse('"1"'), Term(Tag(".String."), "1", None))
self.assertRaises(ValueError, parse, "'x'(x)")
self.assertRaises(ValueError, parse, '3.14(1)')
self.assertRaises(ValueError, parse, '"foo"(x)')
self.assertRaises(ValueError, parse, "1(2)")
def composite(tagName, data, span):
"""
Create a token with a name and data.
"""
return Term(Tag(tagName), data, None, span)
def test_string(self):
self.assertEqual(lex('"foo\\\nbar"'), [Term(Tag(".String."), 'foobar', None, None)])
self.assertEqual(lex('"foo"'), [Term(Tag(".String."), 'foo', None, None)])
self.assertEqual(lex('"foo bar 9"'), [Term(Tag(".String."), 'foo bar 9', None, None)])
self.assertEqual(lex('"foo\\nbar"'), [Term(Tag(".String."), 'foo\nbar', None, None)])
def test_vertical(self):
self.assertEqual(lex(VERTICAL_SPACE),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag(':'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "blee", None, None),
Term(Tag('EOL'), None, None, None)])
def leafTag(tagName, span):
"""
Create a token with no data.
"""
return Term(Tag(tagName), None, None, span)
def test_colon(self):
self.assertEqual(lex(':x'), [Term(Tag(':'), None, None, None),
Term(Tag('IDENTIFIER'), 'x', None, None)])
self.assertEqual(lex(':='), [Term(Tag(':='), None, None, None)])
self.assertEqual(lex('::'), [Term(Tag('::'), None, None, None)])
def test_crunch(self):
self.assertEqual(lex('<'), [Term(Tag('<'), None, None, None)])
self.assertEqual(lex('<-'), [Term(Tag('<-'), None, None, None)])
self.assertEqual(lex('<='), [Term(Tag('<='), None, None, None)])
self.assertEqual(lex('<<='), [Term(Tag('<<='), None, None, None)])
self.assertEqual(lex('<=>'), [Term(Tag('<=>'), None, None, None)])
def test_plus(self):
self.assertEqual(lex('+'), [Term(Tag('+'), None, None, None)])
self.assertEqual(lex('+='), [Term(Tag('+='), None, None, None)])
def test_minus(self):
self.assertEqual(lex('-'), [Term(Tag('-'), None, None, None)])
self.assertEqual(lex('-='), [Term(Tag('-='), None, None, None)])
self.assertEqual(lex('-> {'), [Term(Tag('->'), None, None, None),
Term(Tag('{'), None, None, None)])
def test_dot(self):
self.assertEqual(lex('.'), [Term(Tag('.'), None, None, None)])
self.assertEqual(lex('..'), [Term(Tag('..'), None, None, None)])
self.assertEqual(lex('..!'), [Term(Tag('..!'), None, None, None)])
def test_caret(self):
self.assertEqual(lex('^'), [Term(Tag('^'), None, None, None)])
self.assertEqual(lex('^='), [Term(Tag('^='), None, None, None)])
def test_multi(self):
self.assertEqual(lex(MULTI_INDENT),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag(':'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag(':'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "biz", None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "blee", None, None),
Term(Tag('EOL'), None, None, None),])
def test_integer(self):
self.assertEqual(lex('0'), [Term(Tag(".int."), 0, None, None)])
self.assertEqual(lex('7'), [Term(Tag(".int."), 7, None, None)])
self.assertEqual(lex('3_000'), [Term(Tag(".int."), 3000, None, None)])
self.assertEqual(lex('0xABad1dea'), [Term(Tag(".int."), 0xABad1dea, None, None)])
def test_uri(self):
self.assertEqual(lex('<unsafe>'),[Term(Tag("URI_GETTER"), "unsafe", None, None)])
self.assertEqual(lex('<import:foo.makeBaz>'),
[Term(Tag("URI"), "import:foo.makeBaz", None, None)])
def test_float(self):
self.assertEqual(lex('1e9'), [Term(Tag(".float64."), 1e9, None, None)])
self.assertEqual(lex('3.1415E17'), [Term(Tag(".float64."), 3.1415E17, None, None)])
self.assertEqual(lex('0.91'), [Term(Tag(".float64."), 0.91, None, None)])
self.assertEqual(lex('3e-2'), [Term(Tag(".float64."), 3e-2, None, None)])
def test_continuation(self):
self.assertEqual(lex(CONTINUATION),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag('('), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "biz", None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag(')'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "blee", None, None),
Term(Tag('EOL'), None, None, None)])