def test_formattedReporting(self): """ Parse errors can be formatted into a nice human-readable view containing the erroneous input and possible fixes. """ g = self.compile(""" dig = '1' | '2' | '3' bits = <dig>+ """) input = "123x321" e = self.assertRaises(ParseError, g.bits, input) self.assertEqual(e.formatError(), dedent(""" 123x321 ^ Parse error at line 1, column 3: expected one of '1', '2', or '3'. trail: [dig] """)) input = "foo\nbaz\nboz\ncharlie\nbuz" e = ParseError(input, 12, expected('token', 'foo') + expected(None, 'b')) self.assertEqual(e.formatError(), dedent(""" charlie ^ Parse error at line 4, column 0: expected one of 'b', or token 'foo'. trail: [] """)) input = '123x321' e = ParseError(input, 3, expected('digit')) self.assertEqual(e.formatError(), dedent(""" 123x321 ^ Parse error at line 1, column 3: expected a digit. trail: [] """))
def test_orErrorTie(self): """ When branches of L{OMetaBase._or} produce errors that tie for rightmost position, they are merged. """ data = "foozle" o = OMetaBase(data) v, e = o._or([lambda: o.token("fog"), lambda: o.token("foz"), lambda: o.token("f")]) self.assertEqual(e[0], 2) self.assertEqual(e[1], [expected("token", "fog")[0], expected("token", "foz")[0]])
def parse_Token(self, spec): """ Consume leading whitespace then the given string. """ val = ' ' while val.isspace(): try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() if val.isspace(): self.input = self.input.tail() wanted = spec.data result = [] for c in wanted: try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() result.append(val) if val == c: self.input = self.input.tail() else: self.error(''.join(result), expected(None, wanted)) yield ''.join(result), p
def parse_Token(self, spec): """ Consume leading whitespace then the given string. """ val = ' ' while val.isspace(): try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() if val.isspace(): self.input = self.input.tail() wanted = spec.data result = [] for c in wanted: try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() result.append(val) if val == c: self.input = self.input.tail() else: raise self.err(p.withMessage(expected("token", wanted))) yield ''.join(result), p
def test_orErrorTie(self): """ When branches of L{OMetaBase._or} produce errors that tie for rightmost position, they are merged. """ data = "foozle" o = OMetaBase(data) v, e = o._or([ lambda: o.token("fog"), lambda: o.token("foz"), lambda: o.token("f") ]) self.assertEqual(e[0], 2) self.assertEqual( e[1], [expected("token", "fog")[0], expected("token", "foz")[0]])
def test_letter(self): """ L{OMetaBase.rule_letter} matches letters. """ o = OMetaBase("a1") v, e = o.rule_letter() self.assertEqual((v, e), ("a", ParseError(o.input, 0, None))) exc = self.assertRaises(ParseError, o.rule_letter) self.assertEqual(exc, ParseError(o.input, 1, expected("letter")))
def test_digit(self): """ L{OMetaBase.rule_digit} matches digits. """ o = OMetaBase("1a") v, e = o.rule_digit() self.assertEqual((v, e), ("1", ParseError("1a", 0, None))) exc = self.assertRaises(ParseError, o.rule_digit) self.assertEqual(exc, ParseError(o.input, 1, expected("digit")))
def rule_tok(self, tok): """ Match a single token from the token stream. """ candidate, e = self.input.head() if candidate.tag.name != tok: raise self.input.nullError(expected("token", tok)) self.input = self.input.tail() return candidate.data, e
def test_tokenFailed(self): """ On failure, L{OMetaBase.rule_token} produces an error indicating the position where match failure occurred and the expected character. """ data = "foozle" o = OMetaBase(data) exc = self.assertRaises(ParseError, o.rule_token, "fog") self.assertEqual(exc.args[0], 2) self.assertEqual(exc.args[1], expected("token", "fog"))
def test_letter(self): """ L{OMetaBase.rule_letter} matches letters. """ o = OMetaBase("a1") v, e = o.rule_letter() self.assertEqual((v, e), ("a", ParseError(o.input, 0, None))) with self.assertRaises(ParseError) as e: o.rule_letter() self.assertEqual(e.exception, ParseError(o.input, 1, expected("letter")))
def test_label2(self): """ Custom labels change the 'expected' in the raised exceptions. """ label = 'lots of xs' g = self.compile("xs = ('x'*) ^ (" + label + ")") self.assertEqual(g.xs(""), "") self.assertEqual(g.xs("x"), "x") self.assertEqual(g.xs("xxx"), "xxx") e = self.assertRaises(ParseError, g.xs, "xy") self.assertEqual(e, ParseError(0, 1, expected(label)).withMessage([("Custom Exception:", label, None)]))
def test_many(self): """ L{OMetaBase.many} returns a list of parsed values and the error that caused the end of the loop. """ data = "ooops" o = OMetaBase(data) self.assertEqual(o.many(lambda: o.rule_exactly('o')), (['o'] * 3, ParseError(o.input, 3, expected(None, 'o'))))
def test_many(self): """ L{OMetaBase.many} returns a list of parsed values and the error that caused the end of the loop. """ data = "ooops" o = OMetaBase(data) self.assertEqual( o.many(lambda: o.rule_exactly('o')), (['o'] * 3, ParseError(o.input, 3, expected(None, 'o'))))
def test_letterOrDigit(self): """ L{OMetaBase.rule_letterOrDigit} matches alphanumerics. """ o = OMetaBase("a1@") v, e = o.rule_letterOrDigit() self.assertEqual((v, e), ("a", ParseError(None, 0, None))) v, e = o.rule_letterOrDigit() self.assertEqual((v, e), ("1", ParseError(None, 1, None))) exc = self.assertRaises(ParseError, o.rule_letterOrDigit) self.assertEqual(exc, ParseError(o.input, 2, expected("letter or digit")))
def test_orFalseSuccess(self): """ When a failing branch of L{OMetaBase._or} gets further than a succeeding one, its error is returned instead of the success branch's. """ data = "foozle" o = OMetaBase(data) v, e = o._or([lambda: o.token("fog"), lambda: o.token("foozik"), lambda: o.token("f")]) self.assertEqual(e[0], 4) self.assertEqual(e[1], expected("token", "foozik"))
def test_letterOrDigit(self): """ L{OMetaBase.rule_letterOrDigit} matches alphanumerics. """ o = OMetaBase("a1@") v, e = o.rule_letterOrDigit() self.assertEqual((v, e), ("a", ParseError(None, 0, None))) v, e = o.rule_letterOrDigit() self.assertEqual((v, e), ("1", ParseError(None, 1, None))) with self.assertRaises(ParseError) as e: o.rule_letterOrDigit() self.assertEqual(e.exception, ParseError(o.input, 2, expected("letter or digit")))
def test_exactlyFail(self): """ L{OMetaBase.rule_exactly} raises L{ParseError} when the requested item doesn't match the input. The error contains info on what was expected and the position. """ data = "foo" o = OMetaBase(data) exc = self.assertRaises(ParseError, o.rule_exactly, "g") self.assertEquals(exc.args[1], expected(None, "g")) self.assertEquals(exc.args[0], 0)
def test_label(self): """ Custom labels change the 'expected' in the raised exceptions. """ label = 'Letter not starting with digit' g = self.compile("ident = (<letter (letter | digit)*>) ^ (" + label + ")") self.assertEqual(g.ident("a"), "a") self.assertEqual(g.ident("abc"), "abc") self.assertEqual(g.ident("a1z"), "a1z") e = self.assertRaises(ParseError, g.ident, "1a") self.assertEqual(e, ParseError(0, 0, expected(label)).withMessage([("Custom Exception:", label, None)]))
def test_orSimpleFailure(self): """ When none of the alternatives passed to L{OMetaBase._or} succeed, the one that got the furthest is returned. """ data = "foozle" o = OMetaBase(data) with self.assertRaises(ParseError) as e: o._or([lambda: o.token("fog"), lambda: o.token("foozik"), lambda: o.token("woozle")]) self.assertEqual(e.exception[0], 4) self.assertEqual(e.exception[1], expected("token", "foozik"))
def test_label(self): """ L{OMetaBase.label} returns a list of parsed values and the error that caused the end of the loop. """ data = "ooops" label = 'CustomLabel' o = OMetaBase(data) exc = self.assertRaises( ParseError, o.label, lambda: o.rule_exactly('x'), label) self.assertEqual(exc, ParseError(o.input, 0, expected(label)).withMessage([("Custom Exception:", label, None)]))
def rule_letter(self): """ Match a single letter. """ try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() if val in string.ascii_letters: self.input = self.input.tail() yield val, p else: raise self.err(p.withMessage(expected("letter")))
def rule_digit(self): """ Match a digit. """ try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() if val in string.digits: self.input = self.input.tail() yield val, p else: raise self.err(p.withMessage(expected("digit")))
def rule_digit(self): """ Match a digit. """ try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() if val in string.digits: self.input = self.input.tail() yield val, p else: self.error(val, expected(None, "a digit"))
def rule_letter(self): """ Match a single letter. """ try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() if val in string.letters: self.input = self.input.tail() yield val, p else: raise self.err(p.withMessage(expected("letter")))
def rule_letter(self): """ Match a single letter. """ try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() if val in string.letters: self.input = self.input.tail() yield val, p else: self.error(val, expected(None, "a letter"))
def test_orSimpleFailure(self): """ When none of the alternatives passed to L{OMetaBase._or} succeed, the one that got the furthest is returned. """ data = "foozle" o = OMetaBase(data) exc = self.assertRaises(ParseError, o._or, [ lambda: o.token("fog"), lambda: o.token("foozik"), lambda: o.token("woozle") ]) self.assertEqual(exc.args[0], 4) self.assertEqual(exc.args[1], expected("token", "foozik"))
def test_orFalseSuccess(self): """ When a failing branch of L{OMetaBase._or} gets further than a succeeding one, its error is returned instead of the success branch's. """ data = "foozle" o = OMetaBase(data) v, e = o._or([ lambda: o.token("fog"), lambda: o.token("foozik"), lambda: o.token("f") ]) self.assertEqual(e[0], 4) self.assertEqual(e[1], expected("token", "foozik"))
def test_label(self): """ L{OMetaBase.label} returns a list of parsed values and the error that caused the end of the loop. """ data = "ooops" label = 'CustomLabel' o = OMetaBase(data) with self.assertRaises(ParseError) as e: o.label(lambda: o.rule_exactly('x'), label) self.assertEqual( e.exception, ParseError(o.input, 0, expected(label)).withMessage([ ("Custom Exception:", label, None) ]))
def parse_Exactly(self, spec): """ Accept a one or more characters that equal the given spec. """ wanted = spec.data result = [] for c in wanted: try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() result.append(val) if val == c: self.input = self.input.tail() else: raise self.err(p.withMessage(expected(None, wanted))) yield ''.join(result), p
def parse_Exactly(self, spec): """ Accept a one or more characters that equal the given spec. """ wanted = spec.data result = [] for c in wanted: try: val, p = self.input.head() except EOFError: yield _feed_me val, p = self.input.head() result.append(val) if val == c: self.input = self.input.tail() else: self.error(''.join(result), expected(None, wanted)) yield ''.join(result), p
def rule_until(self, rule, token): """ Parse up until a given token, using the given rule. The token may be multiple characters or a single character. """ m = self.input try: result = [] while True: try: s = self.input for char in token: v, e = self.exactly(char) return result, e except ParseError: self.input = s v, e = self.apply(rule) result.append(v) except ParseError, pe: self.input = m raise pe.withMessage(expected("%s until" % rule, token))
return v, err elif name == "Predicate": val, err = self._eval(run, args[0]) if not val: raise err else: return True, err elif name == "List": v, e = run.rule_anything() oldInput = run.input try: run.input = InputStream.fromIterable(v) except TypeError: raise e.withMessage(expected("an iterable")) self._eval(run, args[0]) run.end() run.input = oldInput return v, e elif name in ("Action", "Python"): lo = self._localsStack[-1] val = eval(args[0].data, self._globals, lo) return (val, run.input.nullError()) elif name == "ConsumedBy": oldInput = run.input _, err = self._eval(run, args[0]) slice = oldInput.data[oldInput.position:run.input.position] return slice, err
def _eval(self, run, expr): name = expr.tag.name args = expr.args if name == "Apply": ruleName = args[0].data return self._apply(run, ruleName, args[2].args) elif name == "Exactly": return run.exactly(args[0].data) elif name == "Label": label = args[1].data try: val, err = self._eval(run, args[0]) return val, err.withMessage([("Custom Exception:", label, None) ]) except ParseError as err: e = err raise e.withMessage([("Custom Exception:", label, None)]) elif name == "Token": if run.tree: return run._apply(run.rule_exactly, "exactly", [args[0].data]) else: return run._apply(run.rule_token, "token", [args[0].data]) elif name in ("Many", "Many1"): ans = [self._eval(run, args[0])[0]] if name == "Many1" else [] err = None while True: try: m = run.input v, _ = self._eval(run, args[0]) ans.append(v) except ParseError as e: err = e run.input = m break return ans, err elif name == "Repeat": if args[0].tag.name == '.int.': min = args[0].data else: min = self._localsStack[-1][args[0].data] if args[1].tag.name == '.int.': max = args[1].data elif args[1].tag.name == 'null': max = None else: max = self._localsStack[-1][args[1].data] if min == max == 0: return "", None ans = [] e = None for i in range(min): v, e = self._eval(run, args[2]) ans.append(v) for i in range(min, max): try: m = run.input v, e = self._eval(run, args[2]) ans.append(v) except ParseError as err: e = err run.input = m break return ans, e elif name == "Optional": i = run.input try: return self._eval(run, args[0]) except ParseError: run.input = i return (None, run.input.nullError()) elif name == "Or": errors = [] for e in args[0].args: try: m = run.input x = self._eval(run, e) ret, err = x errors.append(err) return ret, joinErrors(errors) except ParseError as err: errors.append(err) run.input = m raise joinErrors(errors) elif name == "Not": m = run.input try: self._eval(run, args[0]) except ParseError as err: run.input = m return True, run.input.nullError() else: raise run.input.nullError() elif name == "Lookahead": try: m = run.input return self._eval(run, args[0]) finally: run.input = m elif name == "And": v = None, run.input.nullError() for e in args[0].args: v = self._eval(run, e) return v elif name == "Bind": v, err = self._eval(run, args[1]) if args[0].data: self._localsStack[-1][args[0].data] = v else: for n, val in zip(args[0].args, v): self._localsStack[-1][n.data] = val return v, err elif name == "Predicate": val, err = self._eval(run, args[0]) if not val: raise err else: return True, err elif name == "List": v, e = run.rule_anything() oldInput = run.input try: run.input = InputStream.fromIterable(v) except TypeError: raise e.withMessage(expected("an iterable")) self._eval(run, args[0]) run.end() run.input = oldInput return v, e elif name in ("Action", "Python"): lo = self._localsStack[-1] val = eval(args[0].data, self._globals, lo) return (val, run.input.nullError()) elif name == "ConsumedBy": oldInput = run.input _, err = self._eval(run, args[0]) slice = oldInput.data[oldInput.position:run.input.position] return slice, err else: raise ValueError("Unrecognized term: %r" % (name, ))
def _eval(self, run, expr): name = expr.tag.name args = expr.args if name == "Apply": ruleName = args[0].data return self._apply(run, ruleName, args[2].args) elif name == "Exactly": return run.exactly(args[0].data) elif name == "Label": label = args[1].data try: val, err = self._eval(run, args[0]) return val, err.withMessage([("Custom Exception:", label, None)]) except ParseError as err: e=err raise e.withMessage([("Custom Exception:", label, None)]) elif name == "Token": if run.tree: return run._apply(run.rule_exactly, "exactly", [args[0].data]) else: return run._apply(run.rule_token, "token", [args[0].data]) elif name in ("Many", "Many1"): ans = [self._eval(run, args[0])[0]] if name == "Many1" else [] err = None while True: try: m = run.input v, _ = self._eval(run, args[0]) ans.append(v) except ParseError as e: err = e run.input = m break return ans, err elif name == "Repeat": if args[0].tag.name == '.int.': min = args[0].data else: min = self._localsStack[-1][args[0].data] if args[1].tag.name == '.int.': max = args[1].data elif args[1].tag.name == 'null': max = None else: max = self._localsStack[-1][args[1].data] if min == max == 0: return "", None ans = [] e = None for i in range(min): v, e = self._eval(run, args[2]) ans.append(v) for i in range(min, max): try: m = run.input v, e = self._eval(run, args[2]) ans.append(v) except ParseError as err: e = err run.input = m break return ans, e elif name == "Optional": i = run.input try: return self._eval(run, args[0]) except ParseError: run.input = i return (None, run.input.nullError()) elif name == "Or": errors = [] for e in args[0].args: try: m = run.input x = self._eval(run, e) ret, err = x errors.append(err) return ret, joinErrors(errors) except ParseError as err: errors.append(err) run.input = m raise joinErrors(errors) elif name == "Not": m = run.input try: self._eval(run, args[0]) except ParseError as err: run.input = m return True, run.input.nullError() else: raise run.input.nullError() elif name == "Lookahead": try: m = run.input return self._eval(run, args[0]) finally: run.input = m elif name == "And": v = None, run.input.nullError() for e in args[0].args: v = self._eval(run, e) return v elif name == "Bind": v, err = self._eval(run, args[1]) if args[0].data: self._localsStack[-1][args[0].data] = v else: for n, val in zip(args[0].args, v): self._localsStack[-1][n.data] = val return v, err elif name == "Predicate": val, err = self._eval(run, args[0]) if not val: raise err else: return True, err elif name == "List": v, e = run.rule_anything() oldInput = run.input try: run.input = InputStream.fromIterable(v) except TypeError: raise e.withMessage(expected("an iterable")) self._eval(run, args[0]) run.end() run.input = oldInput return v, e elif name in ("Action", "Python"): lo = self._localsStack[-1] val = eval(args[0].data, self._globals, lo) return (val, run.input.nullError()) elif name == "ConsumedBy": oldInput = run.input _, err = self._eval(run, args[0]) slice = oldInput.data[oldInput.position:run.input.position] return slice, err else: raise ValueError("Unrecognized term: %r" % (name,))