def testLogicalExpression(self): repository = {'TrueFalse':load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")} productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/LogicalExpression.bnf", repository) parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical) tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "True&&False")] result = parser.get_trees(tokens) self.assertTrue(result) result = parser.get_trees("True&|False") self.assertFalse(result)
def __init__(self, bnf, parser=None): Checker.__init__(self) self.gd = bnf parser = bnf.options.get("parser", parser) if parser in ("descent", "auto", "default", None): from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser self.__parser = BacktracingErrorRecursiveDescentParser(bnf) else: raise ValueError("Unknown parser : " + parser)
def testLogicalExp(self): repository = {'TrueFalse':load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")} productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/LogicalExpression.bnf", repository) parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical) tokens = [x for x in lex(repository['TrueFalse'].alphabet, ascii_encoding, self.tokelist5)] self.assertEqual(len(tokens), 1) #tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, Encoding('ascii'), tokens)] #FIXME tokens = [Token('True', repository['TrueFalse'])] result = parser.get_trees(tokens) self.assertTrue(result)
def testTrueFalse(self): productionrulesetlogical = load_bnf_file( "pydsl/contrib/grammar/TrueFalse.bnf") parser = BacktracingErrorRecursiveDescentParser( productionrulesetlogical) tokens = [ x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, self.tokelist5) ] result = parser.get_trees(tokens) self.assertTrue(result)
def testHTMLTable(self): repository = {'integer':RegularExpression("^[0123456789]*$")} productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/TrueHTMLTable.bnf", repository) parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical) lexed = lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><tr><td>1</td></tr></table>") self.assertTrue(lexed) result = parser.get_trees(lexed) self.assertTrue(result) lexed = [x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><td>1</td></tr></table>")] result = parser.get_trees(lexed) self.assertFalse(result)
def testLogicalExpression(self): repository = { 'TrueFalse': load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf") } productionrulesetlogical = load_bnf_file( "pydsl/contrib/grammar/LogicalExpression.bnf", repository) parser = BacktracingErrorRecursiveDescentParser( productionrulesetlogical) tokens = [ x[0] for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "True&&False") ] result = parser.get_trees(tokens) self.assertTrue(result) result = parser.get_trees("True&|False") self.assertFalse(result)
class BNFChecker(Checker): """Calls another program to perform checking. Args are always file names""" def __init__(self, bnf, parser = None): Checker.__init__(self) self.gd = bnf parser = bnf.options.get("parser", parser) if parser in ("descent", "auto", "default", None): from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser self.__parser = BacktracingErrorRecursiveDescentParser(bnf) else: raise ValueError("Unknown parser : " + parser) def check(self, data): if isinstance(data, str): from pydsl.token import PositionToken from pydsl.encoding import ascii_encoding data = [PositionToken(x, ascii_encoding, i, i+1) for i,x in enumerate(data)] if not isinstance(data, Iterable): raise TypeError(data) if not all(check(self.gd.alphabet, [x]) for x in data): LOG.warning("Invalid input: %s,%s" % (self.gd.alphabet, data)) return False try: return len(self.__parser.get_trees(data)) > 0 except IndexError: return False
class BNFChecker(Checker): """Calls another program to perform checking. Args are always file names""" def __init__(self, bnf, parser=None): Checker.__init__(self) self.gd = bnf parser = bnf.options.get("parser", parser) if parser in ("descent", "auto", "default", None): from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser self.__parser = BacktracingErrorRecursiveDescentParser(bnf) else: raise ValueError("Unknown parser : " + parser) def check(self, data): if isinstance(data, str): from pydsl.token import PositionToken from pydsl.encoding import ascii_encoding data = [ PositionToken(x, ascii_encoding, i, i + 1) for i, x in enumerate(data) ] if not isinstance(data, Iterable): raise TypeError(data) if not all(check(self.gd.alphabet, [x]) for x in data): LOG.warning("Invalid input: %s,%s" % (self.gd.alphabet, data)) return False try: return len(self.__parser.get_trees(data)) > 0 except IndexError: return False
def testLogicalExp(self): repository = { 'TrueFalse': load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf") } productionrulesetlogical = load_bnf_file( "pydsl/contrib/grammar/LogicalExpression.bnf", repository) parser = BacktracingErrorRecursiveDescentParser( productionrulesetlogical) tokens = [ x for x in lex(repository['TrueFalse'].alphabet, ascii_encoding, self.tokelist5) ] self.assertEqual(len(tokens), 1) #tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, Encoding('ascii'), tokens)] #FIXME tokens = [Token('True', repository['TrueFalse'])] result = parser.get_trees(tokens) self.assertTrue(result)
def testHTMLTable(self): repository = {'integer': RegularExpression("^[0123456789]*$")} productionrulesetlogical = load_bnf_file( "pydsl/contrib/grammar/TrueHTMLTable.bnf", repository) parser = BacktracingErrorRecursiveDescentParser( productionrulesetlogical) lexed = lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><tr><td>1</td></tr></table>") self.assertTrue(lexed) result = parser.get_trees(lexed) self.assertTrue(result) lexed = [ x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><td>1</td></tr></table>") ] result = parser.get_trees(lexed) self.assertFalse(result)
def testRecursiveDescentParserNullBad(self): descentparser = BacktracingErrorRecursiveDescentParser(productionset2) from pydsl.encoding import ascii_encoding ascii_encoding = ascii_encoding lexed_string4 = lex(productionset2.alphabet, ascii_encoding, string4) result = descentparser(lexed_string4) self.assertFalse(result) result = descentparser(list(string4)) self.assertFalse(result)
def __init__(self, bnf, parser = None): Checker.__init__(self) self.gd = bnf parser = bnf.options.get("parser", parser) if parser in ("descent", "auto", "default", None): from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser self.__parser = BacktracingErrorRecursiveDescentParser(bnf) else: raise ValueError("Unknown parser : " + parser)
def test_main_case(self): input_data = "1+2" ascii_lexer = lexer_factory(ascii_encoding, None) ascii_tokens = [x for x in ascii_lexer(input_data)] self.assertListEqual([str(x) for x in ascii_tokens], ['1', '+', '2']) def concept_translator_fun(inputtokens): result = [] for x in inputtokens: if str(x) == "1": result.append("one") elif str(x) == "2": result.append("two") elif str(x) == "+": result.append("addition") else: raise Exception(x.__class__.__name__) return result def to_number(number): if number == "one": return 1 if number == "two": return 2 math_expression_concepts = concept_translator_fun(ascii_tokens) self.assertListEqual(math_expression_concepts, ['one', 'addition', 'two']) grammar_def = [ "S ::= E", "E ::= one addition two", "one := String,one", "two := String,two", "addition := String,addition", ] from pydsl.file.BNF import strlist_to_production_set production_set = strlist_to_production_set(grammar_def, {}) from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser rdp = BacktracingErrorRecursiveDescentParser(production_set) parse_tree = rdp(math_expression_concepts) from pydsl.grammar.symbol import NonTerminalSymbol def parse_tree_walker(tree): if tree.symbol == NonTerminalSymbol("S"): return parse_tree_walker(tree.childlist[0]) if tree.symbol == NonTerminalSymbol("E"): return to_number(tree.childlist[0].symbol.gd) + to_number( tree.childlist[2].symbol.gd) raise Exception result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 3)
def parser_factory(grammar, parser = None): from pydsl.grammar.BNF import BNFGrammar if isinstance(grammar, BNFGrammar): if parser in ("auto" , "default" , "descent", None): from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser return BacktracingErrorRecursiveDescentParser(grammar) elif parser == "lr0": from pydsl.parser.LR0 import LR0Parser return LR0Parser(grammar) elif parser == "ll1": from pydsl.parser.LL import LL1RecursiveDescentParser return LL1RecursiveDescentParser(grammar) else: raise Exception("Wrong parser name: " + str(parser)) else: raise ValueError(grammar)
class BNFChecker(Checker): """Calls another program to perform checking. Args are always file names""" def __init__(self, bnf, parser = None): Checker.__init__(self) self.gd = bnf parser = bnf.options.get("parser",parser) if parser in ("descent", "auto", "default", None): from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser self.__parser = BacktracingErrorRecursiveDescentParser(bnf) else: raise ValueError("Unknown parser : " + parser) def check(self, data): for element in data: if not check(self.gd.alphabet, element): LOG.warning("Invalid input: %s,%s" % (self.gd.alphabet, element)) return False try: return len(self.__parser.get_trees(data)) > 0 except IndexError: return False
def testRecursiveLeftRecursion(self): descentparser = BacktracingErrorRecursiveDescentParser(productionsetlr) self.assertRaises(RuntimeError, descentparser, dots)
def testRecursiveDescentParserStore(self): descentparser = BacktracingErrorRecursiveDescentParser(productionset1) result = descentparser(string1) self.assertTrue(result) result = descentparser(list(string1)) self.assertTrue(result)
def testRecursiveDescentParserBad(self): descentparser = BacktracingErrorRecursiveDescentParser(productionset1) result = descentparser(string2) self.assertFalse(result) result = descentparser(list(string2)) self.assertFalse(result)
def testRecursiveDescentParserNull(self): descentparser = BacktracingErrorRecursiveDescentParser(productionset2) result = descentparser(string3) self.assertTrue(result) result = descentparser(list(string3)) self.assertTrue(result)
def testTrueFalse(self): productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf") parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical) tokens = [x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, self.tokelist5)] result = parser.get_trees(tokens) self.assertTrue(result)
def testCenterRecursion(self): descentparser = BacktracingErrorRecursiveDescentParser(productionsetcr) result = descentparser(dots) self.assertTrue(result) result = descentparser(list(dots)) self.assertTrue(result)