def test_braces(self):
self.assertEqual(lex('[a, 1]'),
[Term(Tag('['), None, None, None),
Term(Tag('IDENTIFIER'), 'a', None, None),
Term(Tag(','), None, None, None),
Term(Tag('.int.'), 1, None, None),
Term(Tag(']'), None, None, None)])
def test_dedent(self):
self.assertEqual(lex(SIMPLE_DEDENT),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag(':'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "blee", None, None),
Term(Tag('EOL'), None, None, None)])
def test_holes(self):
self.assertEqual(lex('${'), [Term(Tag('${'), None, None, None)])
self.assertEqual(lex('$blee'), [Term(Tag('DOLLAR_IDENT'), "blee", None, None)])
self.assertEqual(lex('@{'), [Term(Tag('@{'), None, None, None)])
self.assertEqual(lex('@fred'), [Term(Tag('AT_IDENT'), "fred", None, None)])
self.assertEqual(lex('@_fred'), [Term(Tag('AT_IDENT'), "_fred", None, None)])
self.assertEqual(lex('@_'), [Term(Tag('AT_IDENT'), "_", None, None)])
self.assertEqual(lex('$0'), [Term(Tag('$'), None, None, None),
Term(Tag(".int."), 0, None, None)])
self.assertEqual(lex('@1'), [Term(Tag('@'), None, None, None),
Term(Tag(".int."), 1, None, None)])
def test_arrow(self):
self.assertEqual(lex(ARROW_INDENT),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag('->'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('EOL'), None, None, None)])
def generate_Predicate(self, out, term, debugname=None):
"""
Generate a call to self.pred(lambda: expr).
"""
fname = self._newThunkFor(out, "pred", Term(Tag("Action"), None,
[term], None))
return self._expr(out, "pred", "self.pred(%s)" %(fname,), debugname)
def test_simpleTag(self):
"""
Tags are parsed properly.
"""
parse = self.getParser("tag")
self.assertEqual(parse("foo"), Tag("foo"))
self.assertEqual(parse('::"foo"'), Tag('::"foo"'))
self.assertEqual(parse("::foo"), Tag('::foo'))
self.assertEqual(parse("foo::baz"), Tag('foo::baz'))
self.assertEqual(parse('foo::"baz"'), Tag('foo::"baz"'))
self.assertEqual(parse("biz::baz::foo"), Tag('biz::baz::foo'))
self.assertEqual(parse("foo_yay"), Tag('foo_yay'))
self.assertEqual(parse("foo$baz32"), Tag('foo$baz32'))
self.assertEqual(parse("foo-baz.19"), Tag('foo-baz.19'))
def coerceToQuasiMatch(val, isFunctorHole, tag):
if isFunctorHole:
if val is None:
result = Term(Tag("null"), None, None)
elif isinstance(val, Term):
if len(val.args) != 0:
return None
else:
result = val
elif isinstance(val, basestring):
result = Term(Tag(val), None, None)
elif isinstance(val, bool):
result = Term(Tag(["false", "true"][val]), None, None)
else:
return None
else:
result = coerceToTerm(val)
if tag is not None and result.tag != tag:
return None
return result
def test_zap(self):
self.assertEqual(lex('1 >'), [Term(Tag('.int.'), 1, None, None),
Term(Tag('>'), None, None, None)])
self.assertEqual(lex('1 >='), [Term(Tag('.int.'), 1, None, None),
Term(Tag('>='), None, None, None)])
self.assertEqual(lex('1 >>='), [Term(Tag('.int.'), 1, None, None),
Term(Tag('>>='), None, None, None)])
def convertInfix(self, maxPrec):
leftAssociative = set(
['+', '-', '>>', '<<', '/', '*', '//', '%', '%%'])
selfAssociative = set(['|', '&'])
lhs, err = self.rule_prefix()
output = [lhs]
opstack = []
while True:
opTok, _ = self.input.head()
op = opTok.tag.name
if op not in OPERATORS:
break
nextPrec = OPERATORS[op][0]
if nextPrec > maxPrec:
break
self.input = self.input.tail()
self.rule_br()
if opstack and (opstack[-1][0] < nextPrec or op in leftAssociative
and opstack[-1][0] <= nextPrec
or op in selfAssociative and opstack[-1][2] == op):
prec, nodeName, opname = opstack.pop()
rhs = output.pop()
lhs = output.pop()
output.append(Term(Tag(nodeName), None, (lhs, rhs), None))
opstack.append(OPERATORS[op] + (op, ))
if op in ['=~', '!~']:
nextTok, err = self.rule_pattern()
else:
nextTok, err = self.rule_prefix()
output.append(nextTok)
while opstack:
prec, nodeName, opname = opstack.pop()
rhs = output.pop()
lhs = output.pop()
output.append(Term(Tag(nodeName), None, (lhs, rhs), None))
assert len(output) == 1
return output[0], err
def test_simpleTerm(self):
"""
Kernel syntax for terms is parsed properly.
"""
parse = self.getParser("baseTerm")
self.assertEqual(parse("x"), Term(Tag("x"), None, None))
self.assertEqual(parse("x()"), Term(Tag("x"), None, []))
self.assertEqual(parse("x(1)"),
Term(Tag("x"), None, (Term(Tag(".int."), 1, None), )))
self.assertEqual(
parse("x(1, 2)"),
Term(Tag("x"), None,
(Term(Tag(".int."), 1, None), Term(Tag(".int."), 2, None))))
self.assertEqual(parse("1"), Term(Tag(".int."), 1, None))
self.assertEqual(parse('"1"'), Term(Tag(".String."), "1", None))
self.assertRaises(ValueError, parse, "'x'(x)")
self.assertRaises(ValueError, parse, '3.14(1)')
self.assertRaises(ValueError, parse, '"foo"(x)')
self.assertRaises(ValueError, parse, "1(2)")
def test_vertical(self):
self.assertEqual(lex(VERTICAL_SPACE),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag(':'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "blee", None, None),
Term(Tag('EOL'), None, None, None)])
def composite(tagName, data, span):
"""
Create a token with a name and data.
"""
return Term(Tag(tagName), data, None, span)
def leafTag(tagName, span):
"""
Create a token with no data.
"""
return Term(Tag(tagName), None, None, span)
def test_uri(self):
self.assertEqual(lex('<unsafe>'),[Term(Tag("URI_GETTER"), "unsafe", None, None)])
self.assertEqual(lex('<import:foo.makeBaz>'),
[Term(Tag("URI"), "import:foo.makeBaz", None, None)])
def test_literal(self):
"""
Literals are parsed to literal terms.
"""
parse = self.getParser("literal")
self.assertEqual(parse('"foo bar"'),
Term(Tag('.String.'), "foo bar", None, None))
self.assertEqual(parse("'x'"),
Term(Tag('.char.'), 'x', None, None))
self.assertEqual(parse("0xDECAFC0FFEEBAD"),
Term(Tag('.int.'), 0xDECAFC0FFEEBAD, None, None))
self.assertEqual(parse("0755"),
Term(Tag('.int.'), 0o755, None, None))
self.assertEqual(parse("3.14159E17"),
Term(Tag('.float64.'), 3.14159E17, None, None))
self.assertEqual(parse("1e9"),
Term(Tag('.float64.'), 1e9, None, None))
self.assertEqual(parse("0"), Term(Tag(".int."), 0, None, None))
self.assertEqual(parse("7"), Term(Tag(".int."), 7, None, None))
self.assertEqual(parse("-1"), Term(Tag(".int."), -1, None, None))
self.assertEqual(parse("-3.14"),
Term(Tag('.float64.'), -3.14, None, None))
self.assertEqual(parse("3_000"),
Term(Tag('.int.'), 3000, None, None))
self.assertEqual(parse("0.91"),
Term(Tag('.float64.'), 0.91, None, None))
self.assertEqual(parse("3e-2"),
Term(Tag('.float64.'), 3e-2, None, None))
self.assertEqual(parse("'\\n'"),
Term(Tag('.char.'), character("\n"), None, None))
self.assertEqual(parse('"foo\\nbar"'),
Term(Tag('.String.'), "foo\nbar", None, None))
self.assertEqual(parse("'\\u0061'"),
Term(Tag('.char.'), character("a"), None, None))
self.assertEqual(parse('"z\141p"'),
Term(Tag('.String.'), "zap", None, None))
self.assertEqual(parse('"x\41"'),
Term(Tag('.String.'), "x!", None, None))
self.assertEqual(parse('"foo\\\nbar"'),
Term(Tag('.String.'), "foobar", None, None))
def Tuple(args):
return Term(Tag(".tuple."), None, tuple(args))
def test_plus(self):
self.assertEqual(lex('+'), [Term(Tag('+'), None, None, None)])
self.assertEqual(lex('+='), [Term(Tag('+='), None, None, None)])
def test_caret(self):
self.assertEqual(lex('^'), [Term(Tag('^'), None, None, None)])
self.assertEqual(lex('^='), [Term(Tag('^='), None, None, None)])
def test_dot(self):
self.assertEqual(lex('.'), [Term(Tag('.'), None, None, None)])
self.assertEqual(lex('..'), [Term(Tag('..'), None, None, None)])
self.assertEqual(lex('..!'), [Term(Tag('..!'), None, None, None)])
def test_integer(self):
self.assertEqual(lex('0'), [Term(Tag(".int."), 0, None, None)])
self.assertEqual(lex('7'), [Term(Tag(".int."), 7, None, None)])
self.assertEqual(lex('3_000'), [Term(Tag(".int."), 3000, None, None)])
self.assertEqual(lex('0xABad1dea'), [Term(Tag(".int."), 0xABad1dea, None, None)])
def test_continuation(self):
self.assertEqual(lex(CONTINUATION),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag('('), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "biz", None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag(')'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "blee", None, None),
Term(Tag('EOL'), None, None, None)])
def Attr(k, v):
return Term(Tag(".attr."), None, (k, v))
def makeTag(nameSegs):
return Tag('::'.join(nameSegs))
def test_multi(self):
self.assertEqual(lex(MULTI_INDENT),
[Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "foo", None, None),
Term(Tag(':'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "baz", None, None),
Term(Tag(':'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('INDENT'), None, None, None),
Term(Tag('IDENTIFIER'), "biz", None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('DEDENT'), None, None, None),
Term(Tag('EOL'), None, None, None),
Term(Tag('IDENTIFIER'), "blee", None, None),
Term(Tag('EOL'), None, None, None),])
def Bag(args):
return Term(Tag(".bag."), None, tuple(args))
def test_crunch(self):
self.assertEqual(lex('<'), [Term(Tag('<'), None, None, None)])
self.assertEqual(lex('<-'), [Term(Tag('<-'), None, None, None)])
self.assertEqual(lex('<='), [Term(Tag('<='), None, None, None)])
self.assertEqual(lex('<<='), [Term(Tag('<<='), None, None, None)])
self.assertEqual(lex('<=>'), [Term(Tag('<=>'), None, None, None)])
def test_minus(self):
self.assertEqual(lex('-'), [Term(Tag('-'), None, None, None)])
self.assertEqual(lex('-='), [Term(Tag('-='), None, None, None)])
self.assertEqual(lex('-> {'), [Term(Tag('->'), None, None, None),
Term(Tag('{'), None, None, None)])
def test_float(self):
self.assertEqual(lex('1e9'), [Term(Tag(".float64."), 1e9, None, None)])
self.assertEqual(lex('3.1415E17'), [Term(Tag(".float64."), 3.1415E17, None, None)])
self.assertEqual(lex('0.91'), [Term(Tag(".float64."), 0.91, None, None)])
self.assertEqual(lex('3e-2'), [Term(Tag(".float64."), 3e-2, None, None)])
def test_colon(self):
self.assertEqual(lex(':x'), [Term(Tag(':'), None, None, None),
Term(Tag('IDENTIFIER'), 'x', None, None)])
self.assertEqual(lex(':='), [Term(Tag(':='), None, None, None)])
self.assertEqual(lex('::'), [Term(Tag('::'), None, None, None)])
def test_string(self):
self.assertEqual(lex('"foo\\\nbar"'), [Term(Tag(".String."), 'foobar', None, None)])
self.assertEqual(lex('"foo"'), [Term(Tag(".String."), 'foo', None, None)])
self.assertEqual(lex('"foo bar 9"'), [Term(Tag(".String."), 'foo bar 9', None, None)])
self.assertEqual(lex('"foo\\nbar"'), [Term(Tag(".String."), 'foo\nbar', None, None)])
