def test_formattedReporting(self):
"""
Parse errors can be formatted into a nice human-readable view
containing the erroneous input and possible fixes.
"""
g = self.compile("""
dig = '1' | '2' | '3'
bits = <dig>+
""")
input = "123x321"
e = self.assertRaises(ParseError, g.bits, input)
self.assertEqual(e.formatError(),
dedent("""
123x321
^
Parse error at line 1, column 3: expected one of '1', '2', or '3'. trail: [dig]
"""))
input = "foo\nbaz\nboz\ncharlie\nbuz"
e = ParseError(input, 12, expected('token', 'foo') + expected(None, 'b'))
self.assertEqual(e.formatError(),
dedent("""
charlie
^
Parse error at line 4, column 0: expected one of 'b', or token 'foo'. trail: []
"""))
input = '123x321'
e = ParseError(input, 3, expected('digit'))
self.assertEqual(e.formatError(),
dedent("""
123x321
^
Parse error at line 1, column 3: expected a digit. trail: []
"""))
def test_formattedReporting(self):
"""
Parse errors can be formatted into a nice human-readable view
containing the erroneous input and possible fixes.
"""
g = self.compile("""
dig = '1' | '2' | '3'
bits = <dig>+
""")
input = "123x321"
e = self.assertRaises(ParseError, g.bits, input)
self.assertEqual(e.formatError(),
dedent("""
123x321
^
Parse error at line 1, column 3: expected one of '1', '2', or '3'. trail: [dig]
"""))
input = "foo\nbaz\nboz\ncharlie\nbuz"
e = ParseError(input, 12, expected('token', 'foo') + expected(None, 'b'))
self.assertEqual(e.formatError(),
dedent("""
charlie
^
Parse error at line 4, column 0: expected one of 'b', or token 'foo'. trail: []
"""))
input = '123x321'
e = ParseError(input, 3, expected('digit'))
self.assertEqual(e.formatError(),
dedent("""
123x321
^
Parse error at line 1, column 3: expected a digit. trail: []
"""))
def test_orErrorTie(self):
"""
When branches of L{OMetaBase._or} produce errors that tie for rightmost
position, they are merged.
"""
data = "foozle"
o = OMetaBase(data)
v, e = o._or([lambda: o.token("fog"), lambda: o.token("foz"), lambda: o.token("f")])
self.assertEqual(e[0], 2)
self.assertEqual(e[1], [expected("token", "fog")[0], expected("token", "foz")[0]])
def parse_Token(self, spec):
"""
Consume leading whitespace then the given string.
"""
val = ' '
while val.isspace():
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val.isspace():
self.input = self.input.tail()
wanted = spec.data
result = []
for c in wanted:
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
result.append(val)
if val == c:
self.input = self.input.tail()
else:
self.error(''.join(result), expected(None, wanted))
yield ''.join(result), p
def parse_Token(self, spec):
"""
Consume leading whitespace then the given string.
"""
val = ' '
while val.isspace():
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val.isspace():
self.input = self.input.tail()
wanted = spec.data
result = []
for c in wanted:
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
result.append(val)
if val == c:
self.input = self.input.tail()
else:
raise self.err(p.withMessage(expected("token", wanted)))
yield ''.join(result), p
def test_orErrorTie(self):
"""
When branches of L{OMetaBase._or} produce errors that tie for rightmost
position, they are merged.
"""
data = "foozle"
o = OMetaBase(data)
v, e = o._or([
lambda: o.token("fog"), lambda: o.token("foz"),
lambda: o.token("f")
])
self.assertEqual(e[0], 2)
self.assertEqual(
e[1], [expected("token", "fog")[0],
expected("token", "foz")[0]])
def test_letter(self):
"""
L{OMetaBase.rule_letter} matches letters.
"""
o = OMetaBase("a1")
v, e = o.rule_letter()
self.assertEqual((v, e), ("a", ParseError(o.input, 0, None)))
exc = self.assertRaises(ParseError, o.rule_letter)
self.assertEqual(exc, ParseError(o.input, 1, expected("letter")))
def test_digit(self):
"""
L{OMetaBase.rule_digit} matches digits.
"""
o = OMetaBase("1a")
v, e = o.rule_digit()
self.assertEqual((v, e), ("1", ParseError("1a", 0, None)))
exc = self.assertRaises(ParseError, o.rule_digit)
self.assertEqual(exc, ParseError(o.input, 1, expected("digit")))
def rule_tok(self, tok):
"""
Match a single token from the token stream.
"""
candidate, e = self.input.head()
if candidate.tag.name != tok:
raise self.input.nullError(expected("token", tok))
self.input = self.input.tail()
return candidate.data, e
def test_digit(self):
"""
L{OMetaBase.rule_digit} matches digits.
"""
o = OMetaBase("1a")
v, e = o.rule_digit()
self.assertEqual((v, e), ("1", ParseError("1a", 0, None)))
exc = self.assertRaises(ParseError, o.rule_digit)
self.assertEqual(exc, ParseError(o.input, 1, expected("digit")))
def rule_tok(self, tok):
"""
Match a single token from the token stream.
"""
candidate, e = self.input.head()
if candidate.tag.name != tok:
raise self.input.nullError(expected("token", tok))
self.input = self.input.tail()
return candidate.data, e
def test_tokenFailed(self):
"""
On failure, L{OMetaBase.rule_token} produces an error indicating the
position where match failure occurred and the expected character.
"""
data = "foozle"
o = OMetaBase(data)
exc = self.assertRaises(ParseError, o.rule_token, "fog")
self.assertEqual(exc.args[0], 2)
self.assertEqual(exc.args[1], expected("token", "fog"))
def test_tokenFailed(self):
"""
On failure, L{OMetaBase.rule_token} produces an error indicating the
position where match failure occurred and the expected character.
"""
data = "foozle"
o = OMetaBase(data)
exc = self.assertRaises(ParseError, o.rule_token, "fog")
self.assertEqual(exc.args[0], 2)
self.assertEqual(exc.args[1], expected("token", "fog"))
def test_letter(self):
"""
L{OMetaBase.rule_letter} matches letters.
"""
o = OMetaBase("a1")
v, e = o.rule_letter()
self.assertEqual((v, e), ("a", ParseError(o.input, 0, None)))
with self.assertRaises(ParseError) as e:
o.rule_letter()
self.assertEqual(e.exception, ParseError(o.input, 1, expected("letter")))
def test_label2(self):
"""
Custom labels change the 'expected' in the raised exceptions.
"""
label = 'lots of xs'
g = self.compile("xs = ('x'*) ^ (" + label + ")")
self.assertEqual(g.xs(""), "")
self.assertEqual(g.xs("x"), "x")
self.assertEqual(g.xs("xxx"), "xxx")
e = self.assertRaises(ParseError, g.xs, "xy")
self.assertEqual(e, ParseError(0, 1, expected(label)).withMessage([("Custom Exception:", label, None)]))
def test_many(self):
"""
L{OMetaBase.many} returns a list of parsed values and the error that
caused the end of the loop.
"""
data = "ooops"
o = OMetaBase(data)
self.assertEqual(o.many(lambda: o.rule_exactly('o')),
(['o'] * 3, ParseError(o.input, 3,
expected(None, 'o'))))
def test_many(self):
"""
L{OMetaBase.many} returns a list of parsed values and the error that
caused the end of the loop.
"""
data = "ooops"
o = OMetaBase(data)
self.assertEqual(
o.many(lambda: o.rule_exactly('o')),
(['o'] * 3, ParseError(o.input, 3, expected(None, 'o'))))
def test_letterOrDigit(self):
"""
L{OMetaBase.rule_letterOrDigit} matches alphanumerics.
"""
o = OMetaBase("a1@")
v, e = o.rule_letterOrDigit()
self.assertEqual((v, e), ("a", ParseError(None, 0, None)))
v, e = o.rule_letterOrDigit()
self.assertEqual((v, e), ("1", ParseError(None, 1, None)))
exc = self.assertRaises(ParseError, o.rule_letterOrDigit)
self.assertEqual(exc,
ParseError(o.input, 2, expected("letter or digit")))
def test_orFalseSuccess(self):
"""
When a failing branch of L{OMetaBase._or} gets further than a
succeeding one, its error is returned instead of the success branch's.
"""
data = "foozle"
o = OMetaBase(data)
v, e = o._or([lambda: o.token("fog"), lambda: o.token("foozik"), lambda: o.token("f")])
self.assertEqual(e[0], 4)
self.assertEqual(e[1], expected("token", "foozik"))
def test_letterOrDigit(self):
"""
L{OMetaBase.rule_letterOrDigit} matches alphanumerics.
"""
o = OMetaBase("a1@")
v, e = o.rule_letterOrDigit()
self.assertEqual((v, e), ("a", ParseError(None, 0, None)))
v, e = o.rule_letterOrDigit()
self.assertEqual((v, e), ("1", ParseError(None, 1, None)))
with self.assertRaises(ParseError) as e:
o.rule_letterOrDigit()
self.assertEqual(e.exception, ParseError(o.input, 2, expected("letter or digit")))
def test_exactlyFail(self):
"""
L{OMetaBase.rule_exactly} raises L{ParseError} when the requested item
doesn't match the input. The error contains info on what was expected
and the position.
"""
data = "foo"
o = OMetaBase(data)
exc = self.assertRaises(ParseError, o.rule_exactly, "g")
self.assertEquals(exc.args[1], expected(None, "g"))
self.assertEquals(exc.args[0], 0)
def test_label(self):
"""
Custom labels change the 'expected' in the raised exceptions.
"""
label = 'Letter not starting with digit'
g = self.compile("ident = (<letter (letter | digit)*>) ^ (" + label + ")")
self.assertEqual(g.ident("a"), "a")
self.assertEqual(g.ident("abc"), "abc")
self.assertEqual(g.ident("a1z"), "a1z")
e = self.assertRaises(ParseError, g.ident, "1a")
self.assertEqual(e, ParseError(0, 0, expected(label)).withMessage([("Custom Exception:", label, None)]))
def test_exactlyFail(self):
"""
L{OMetaBase.rule_exactly} raises L{ParseError} when the requested item
doesn't match the input. The error contains info on what was expected
and the position.
"""
data = "foo"
o = OMetaBase(data)
exc = self.assertRaises(ParseError, o.rule_exactly, "g")
self.assertEquals(exc.args[1], expected(None, "g"))
self.assertEquals(exc.args[0], 0)
def test_orSimpleFailure(self):
"""
When none of the alternatives passed to L{OMetaBase._or} succeed, the
one that got the furthest is returned.
"""
data = "foozle"
o = OMetaBase(data)
with self.assertRaises(ParseError) as e:
o._or([lambda: o.token("fog"), lambda: o.token("foozik"), lambda: o.token("woozle")])
self.assertEqual(e.exception[0], 4)
self.assertEqual(e.exception[1], expected("token", "foozik"))
def test_label(self):
"""
L{OMetaBase.label} returns a list of parsed values and the error that
caused the end of the loop.
"""
data = "ooops"
label = 'CustomLabel'
o = OMetaBase(data)
exc = self.assertRaises(
ParseError, o.label, lambda: o.rule_exactly('x'), label)
self.assertEqual(exc,
ParseError(o.input, 0, expected(label)).withMessage([("Custom Exception:", label, None)]))
def rule_letter(self):
"""
Match a single letter.
"""
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val in string.ascii_letters:
self.input = self.input.tail()
yield val, p
else:
raise self.err(p.withMessage(expected("letter")))
def rule_digit(self):
"""
Match a digit.
"""
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val in string.digits:
self.input = self.input.tail()
yield val, p
else:
raise self.err(p.withMessage(expected("digit")))
def rule_digit(self):
"""
Match a digit.
"""
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val in string.digits:
self.input = self.input.tail()
yield val, p
else:
raise self.err(p.withMessage(expected("digit")))
def rule_digit(self):
"""
Match a digit.
"""
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val in string.digits:
self.input = self.input.tail()
yield val, p
else:
self.error(val, expected(None, "a digit"))
def rule_digit(self):
"""
Match a digit.
"""
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val in string.digits:
self.input = self.input.tail()
yield val, p
else:
self.error(val, expected(None, "a digit"))
def rule_letter(self):
"""
Match a single letter.
"""
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val in string.letters:
self.input = self.input.tail()
yield val, p
else:
raise self.err(p.withMessage(expected("letter")))
def rule_letter(self):
"""
Match a single letter.
"""
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val in string.letters:
self.input = self.input.tail()
yield val, p
else:
self.error(val, expected(None, "a letter"))
def rule_letter(self):
"""
Match a single letter.
"""
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
if val in string.letters:
self.input = self.input.tail()
yield val, p
else:
self.error(val, expected(None, "a letter"))
def test_orSimpleFailure(self):
"""
When none of the alternatives passed to L{OMetaBase._or} succeed, the
one that got the furthest is returned.
"""
data = "foozle"
o = OMetaBase(data)
exc = self.assertRaises(ParseError, o._or, [
lambda: o.token("fog"), lambda: o.token("foozik"),
lambda: o.token("woozle")
])
self.assertEqual(exc.args[0], 4)
self.assertEqual(exc.args[1], expected("token", "foozik"))
def test_orFalseSuccess(self):
"""
When a failing branch of L{OMetaBase._or} gets further than a
succeeding one, its error is returned instead of the success branch's.
"""
data = "foozle"
o = OMetaBase(data)
v, e = o._or([
lambda: o.token("fog"), lambda: o.token("foozik"),
lambda: o.token("f")
])
self.assertEqual(e[0], 4)
self.assertEqual(e[1], expected("token", "foozik"))
def test_label(self):
"""
L{OMetaBase.label} returns a list of parsed values and the error that
caused the end of the loop.
"""
data = "ooops"
label = 'CustomLabel'
o = OMetaBase(data)
with self.assertRaises(ParseError) as e:
o.label(lambda: o.rule_exactly('x'), label)
self.assertEqual(
e.exception,
ParseError(o.input, 0, expected(label)).withMessage([
("Custom Exception:", label, None)
]))
def parse_Exactly(self, spec):
"""
Accept a one or more characters that equal the given spec.
"""
wanted = spec.data
result = []
for c in wanted:
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
result.append(val)
if val == c:
self.input = self.input.tail()
else:
raise self.err(p.withMessage(expected(None, wanted)))
yield ''.join(result), p
def parse_Exactly(self, spec):
"""
Accept a one or more characters that equal the given spec.
"""
wanted = spec.data
result = []
for c in wanted:
try:
val, p = self.input.head()
except EOFError:
yield _feed_me
val, p = self.input.head()
result.append(val)
if val == c:
self.input = self.input.tail()
else:
self.error(''.join(result), expected(None, wanted))
yield ''.join(result), p
def rule_until(self, rule, token):
"""
Parse up until a given token, using the given rule.
The token may be multiple characters or a single character.
"""
m = self.input
try:
result = []
while True:
try:
s = self.input
for char in token:
v, e = self.exactly(char)
return result, e
except ParseError:
self.input = s
v, e = self.apply(rule)
result.append(v)
except ParseError, pe:
self.input = m
raise pe.withMessage(expected("%s until" % rule, token))
def rule_until(self, rule, token):
"""
Parse up until a given token, using the given rule.
The token may be multiple characters or a single character.
"""
m = self.input
try:
result = []
while True:
try:
s = self.input
for char in token:
v, e = self.exactly(char)
return result, e
except ParseError:
self.input = s
v, e = self.apply(rule)
result.append(v)
except ParseError, pe:
self.input = m
raise pe.withMessage(expected("%s until" % rule, token))
return v, err
elif name == "Predicate":
val, err = self._eval(run, args[0])
if not val:
raise err
else:
return True, err
elif name == "List":
v, e = run.rule_anything()
oldInput = run.input
try:
run.input = InputStream.fromIterable(v)
except TypeError:
raise e.withMessage(expected("an iterable"))
self._eval(run, args[0])
run.end()
run.input = oldInput
return v, e
elif name in ("Action", "Python"):
lo = self._localsStack[-1]
val = eval(args[0].data, self._globals, lo)
return (val, run.input.nullError())
elif name == "ConsumedBy":
oldInput = run.input
_, err = self._eval(run, args[0])
slice = oldInput.data[oldInput.position:run.input.position]
return slice, err
def _eval(self, run, expr):
name = expr.tag.name
args = expr.args
if name == "Apply":
ruleName = args[0].data
return self._apply(run, ruleName, args[2].args)
elif name == "Exactly":
return run.exactly(args[0].data)
elif name == "Label":
label = args[1].data
try:
val, err = self._eval(run, args[0])
return val, err.withMessage([("Custom Exception:", label, None)
])
except ParseError as err:
e = err
raise e.withMessage([("Custom Exception:", label, None)])
elif name == "Token":
if run.tree:
return run._apply(run.rule_exactly, "exactly", [args[0].data])
else:
return run._apply(run.rule_token, "token", [args[0].data])
elif name in ("Many", "Many1"):
ans = [self._eval(run, args[0])[0]] if name == "Many1" else []
err = None
while True:
try:
m = run.input
v, _ = self._eval(run, args[0])
ans.append(v)
except ParseError as e:
err = e
run.input = m
break
return ans, err
elif name == "Repeat":
if args[0].tag.name == '.int.':
min = args[0].data
else:
min = self._localsStack[-1][args[0].data]
if args[1].tag.name == '.int.':
max = args[1].data
elif args[1].tag.name == 'null':
max = None
else:
max = self._localsStack[-1][args[1].data]
if min == max == 0:
return "", None
ans = []
e = None
for i in range(min):
v, e = self._eval(run, args[2])
ans.append(v)
for i in range(min, max):
try:
m = run.input
v, e = self._eval(run, args[2])
ans.append(v)
except ParseError as err:
e = err
run.input = m
break
return ans, e
elif name == "Optional":
i = run.input
try:
return self._eval(run, args[0])
except ParseError:
run.input = i
return (None, run.input.nullError())
elif name == "Or":
errors = []
for e in args[0].args:
try:
m = run.input
x = self._eval(run, e)
ret, err = x
errors.append(err)
return ret, joinErrors(errors)
except ParseError as err:
errors.append(err)
run.input = m
raise joinErrors(errors)
elif name == "Not":
m = run.input
try:
self._eval(run, args[0])
except ParseError as err:
run.input = m
return True, run.input.nullError()
else:
raise run.input.nullError()
elif name == "Lookahead":
try:
m = run.input
return self._eval(run, args[0])
finally:
run.input = m
elif name == "And":
v = None, run.input.nullError()
for e in args[0].args:
v = self._eval(run, e)
return v
elif name == "Bind":
v, err = self._eval(run, args[1])
if args[0].data:
self._localsStack[-1][args[0].data] = v
else:
for n, val in zip(args[0].args, v):
self._localsStack[-1][n.data] = val
return v, err
elif name == "Predicate":
val, err = self._eval(run, args[0])
if not val:
raise err
else:
return True, err
elif name == "List":
v, e = run.rule_anything()
oldInput = run.input
try:
run.input = InputStream.fromIterable(v)
except TypeError:
raise e.withMessage(expected("an iterable"))
self._eval(run, args[0])
run.end()
run.input = oldInput
return v, e
elif name in ("Action", "Python"):
lo = self._localsStack[-1]
val = eval(args[0].data, self._globals, lo)
return (val, run.input.nullError())
elif name == "ConsumedBy":
oldInput = run.input
_, err = self._eval(run, args[0])
slice = oldInput.data[oldInput.position:run.input.position]
return slice, err
else:
raise ValueError("Unrecognized term: %r" % (name, ))
def _eval(self, run, expr):
name = expr.tag.name
args = expr.args
if name == "Apply":
ruleName = args[0].data
return self._apply(run, ruleName, args[2].args)
elif name == "Exactly":
return run.exactly(args[0].data)
elif name == "Label":
label = args[1].data
try:
val, err = self._eval(run, args[0])
return val, err.withMessage([("Custom Exception:", label, None)])
except ParseError as err:
e=err
raise e.withMessage([("Custom Exception:", label, None)])
elif name == "Token":
if run.tree:
return run._apply(run.rule_exactly, "exactly", [args[0].data])
else:
return run._apply(run.rule_token, "token", [args[0].data])
elif name in ("Many", "Many1"):
ans = [self._eval(run, args[0])[0]] if name == "Many1" else []
err = None
while True:
try:
m = run.input
v, _ = self._eval(run, args[0])
ans.append(v)
except ParseError as e:
err = e
run.input = m
break
return ans, err
elif name == "Repeat":
if args[0].tag.name == '.int.':
min = args[0].data
else:
min = self._localsStack[-1][args[0].data]
if args[1].tag.name == '.int.':
max = args[1].data
elif args[1].tag.name == 'null':
max = None
else:
max = self._localsStack[-1][args[1].data]
if min == max == 0:
return "", None
ans = []
e = None
for i in range(min):
v, e = self._eval(run, args[2])
ans.append(v)
for i in range(min, max):
try:
m = run.input
v, e = self._eval(run, args[2])
ans.append(v)
except ParseError as err:
e = err
run.input = m
break
return ans, e
elif name == "Optional":
i = run.input
try:
return self._eval(run, args[0])
except ParseError:
run.input = i
return (None, run.input.nullError())
elif name == "Or":
errors = []
for e in args[0].args:
try:
m = run.input
x = self._eval(run, e)
ret, err = x
errors.append(err)
return ret, joinErrors(errors)
except ParseError as err:
errors.append(err)
run.input = m
raise joinErrors(errors)
elif name == "Not":
m = run.input
try:
self._eval(run, args[0])
except ParseError as err:
run.input = m
return True, run.input.nullError()
else:
raise run.input.nullError()
elif name == "Lookahead":
try:
m = run.input
return self._eval(run, args[0])
finally:
run.input = m
elif name == "And":
v = None, run.input.nullError()
for e in args[0].args:
v = self._eval(run, e)
return v
elif name == "Bind":
v, err = self._eval(run, args[1])
if args[0].data:
self._localsStack[-1][args[0].data] = v
else:
for n, val in zip(args[0].args, v):
self._localsStack[-1][n.data] = val
return v, err
elif name == "Predicate":
val, err = self._eval(run, args[0])
if not val:
raise err
else:
return True, err
elif name == "List":
v, e = run.rule_anything()
oldInput = run.input
try:
run.input = InputStream.fromIterable(v)
except TypeError:
raise e.withMessage(expected("an iterable"))
self._eval(run, args[0])
run.end()
run.input = oldInput
return v, e
elif name in ("Action", "Python"):
lo = self._localsStack[-1]
val = eval(args[0].data, self._globals, lo)
return (val, run.input.nullError())
elif name == "ConsumedBy":
oldInput = run.input
_, err = self._eval(run, args[0])
slice = oldInput.data[oldInput.position:run.input.position]
return slice, err
else:
raise ValueError("Unrecognized term: %r" % (name,))
return v, err
elif name == "Predicate":
val, err = self._eval(run, args[0])
if not val:
raise err
else:
return True, err
elif name == "List":
v, e = run.rule_anything()
oldInput = run.input
try:
run.input = InputStream.fromIterable(v)
except TypeError:
raise e.withMessage(expected("an iterable"))
self._eval(run, args[0])
run.end()
run.input = oldInput
return v, e
elif name in ("Action", "Python"):
lo = self._localsStack[-1]
val = eval(args[0].data, self._globals, lo)
return (val, run.input.nullError())
elif name == "ConsumedBy":
oldInput = run.input
_, err = self._eval(run, args[0])
slice = oldInput.data[oldInput.position:run.input.position]
return slice, err
