def test_calculator_simple(self): grammar_def = [ "S ::= E", "E ::= number operator number", "number := Word,integer,max", "operator := String,+", ] from pydsl.file.BNF import strlist_to_production_set from pydsl.grammar import RegularExpression repository = {'integer':RegularExpression("^[0123456789]*$")} production_set = strlist_to_production_set(grammar_def, repository) rdp = LL1RecursiveDescentParser(production_set) parse_tree = rdp("1+2") def parse_tree_walker(tree): from pydsl.grammar.symbol import NonTerminalSymbol if tree.symbol == NonTerminalSymbol("S"): return parse_tree_walker(tree.childlist[0]) if tree.symbol == NonTerminalSymbol("E"): return int(str(tree.childlist[0].content)) + int(str(tree.childlist[2].content)) else: raise Exception result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 3) from pydsl.grammar.PEG import Choice from pydsl.grammar.definition import String, RegularExpression from pydsl.encoding import ascii_encoding math_alphabet = Choice([RegularExpression("^[0123456789]*$"),Choice([String('+')])]) from pydsl.lex import lex tokens = [x[0] for x in lex(math_alphabet, ascii_encoding, "11+2")] parse_tree = rdp(tokens) result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 13)
def test_main_case(self): input_data = "1+2" ascii_lexer = lexer_factory(ascii_encoding, None) ascii_tokens = [x for x in ascii_lexer(input_data)] self.assertListEqual([str(x) for x in ascii_tokens], ['1', '+', '2']) def concept_translator_fun(inputtokens): result = [] for x in inputtokens: if str(x) == "1": result.append("one") elif str(x) == "2": result.append("two") elif str(x) == "+": result.append("addition") else: raise Exception(x.__class__.__name__) return result def to_number(number): if number == "one": return 1 if number == "two": return 2 math_expression_concepts = concept_translator_fun(ascii_tokens) self.assertListEqual(math_expression_concepts, ['one', 'addition', 'two']) grammar_def = [ "S ::= E", "E ::= one addition two", "one := String,one", "two := String,two", "addition := String,addition", ] from pydsl.file.BNF import strlist_to_production_set production_set = strlist_to_production_set(grammar_def, {}) from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser rdp = BacktracingErrorRecursiveDescentParser(production_set) parse_tree = rdp(math_expression_concepts) from pydsl.grammar.symbol import NonTerminalSymbol def parse_tree_walker(tree): if tree.symbol == NonTerminalSymbol("S"): return parse_tree_walker(tree.childlist[0]) if tree.symbol == NonTerminalSymbol("E"): return to_number(tree.childlist[0].symbol.gd) + to_number( tree.childlist[2].symbol.gd) raise Exception result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 3)
def test_main_case(self): input_data = "1+2" ascii_lexer = lexer_factory(ascii_encoding) ascii_tokens = [x.content for x in ascii_lexer(input_data)] self.assertListEqual([str(x) for x in ascii_tokens], ['1', '+', '2']) def concept_translator_fun(inputtokens): result = [] for x in inputtokens: if x == "1": result.append("one") elif x == "2": result.append("two") elif x == "+": result.append("addition") else: raise Exception(x.__class__.__name__) return result def to_number(number): if number == [x for x in "one"]: return 1 if number == [x for x in "two"]: return 2 math_expression_concepts = concept_translator_fun(ascii_tokens) self.assertListEqual(math_expression_concepts, ['one', 'addition', 'two']) grammar_def = [ "S ::= E", "E ::= one addition two", "one := String,one", "two := String,two", "addition := String,addition", ] from pydsl.file.BNF import strlist_to_production_set production_set = strlist_to_production_set(grammar_def, {}) from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser rdp = BacktracingErrorRecursiveDescentParser(production_set) parse_tree = rdp(math_expression_concepts) from pydsl.grammar.symbol import NonTerminalSymbol def parse_tree_walker(tree): if tree.symbol == NonTerminalSymbol("S"): return parse_tree_walker(tree.childlist[0]) if tree.symbol == NonTerminalSymbol("E"): return to_number(tree.childlist[0].symbol.gd) + to_number(tree.childlist[2].symbol.gd) raise Exception result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 3)
def test_calculator_simple(self): grammar_def = [ "S ::= E", "E ::= number operator number", "number := Word,integer,max", "operator := String,+", ] from pydsl.file.BNF import strlist_to_production_set from pydsl.grammar import RegularExpression repository = {'integer': RegularExpression("^[0123456789]*$")} production_set = strlist_to_production_set(grammar_def, repository) rdp = LL1RecursiveDescentParser(production_set) parse_tree = rdp("1+2") def parse_tree_walker(tree): from pydsl.grammar.symbol import NonTerminalSymbol if tree.symbol == NonTerminalSymbol("S"): return parse_tree_walker(tree.childlist[0]) if tree.symbol == NonTerminalSymbol("E"): return int(str(tree.childlist[0].content)) + int( str(tree.childlist[2].content)) else: raise Exception result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 3) from pydsl.grammar.PEG import Choice from pydsl.grammar.definition import String, RegularExpression from pydsl.encoding import ascii_encoding math_alphabet = Choice( [RegularExpression("^[0123456789]*$"), Choice([String('+')])]) from pydsl.lex import lex tokens = [x for x in lex(math_alphabet, ascii_encoding, "11+2")] parse_tree = rdp(tokens) result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 13)
final2 = NonTerminalSymbol("retrieveexp") final3 = NonTerminalSymbol("exp") rule1 = Production([final1], (symbol1, symbol3, symbol5)) rule2 = Production([final2], (symbol2, symbol3, symbol4)) rule3 = Production([final3], [final1]) rule4 = Production([final3], [final2]) rulelist = (rule1, rule2, rule3, rule4, symbol1, symbol2, symbol3, symbol4, symbol5) productionset1 = BNFGrammar(final3, rulelist) #productionset2 definition symbola = TerminalSymbol(String("A")) symbolb = TerminalSymbol(String("B")) nonterminal = NonTerminalSymbol("res") rulea = Production ((nonterminal,), (symbola, NullSymbol(), symbolb)) productionset2 = BNFGrammar(nonterminal, (rulea, symbola, symbolb)) productionsetlr = strlist_to_production_set(leftrecursive) productionsetrr = strlist_to_production_set(rightrecursive) productionsetcr = strlist_to_production_set(centerrecursive) #arithmetic arithmetic=["E ::= E plus T | T", "T ::= T times F | F" ,"F ::= open_parenthesis E close_parenthesis | id", "id := String,123" , "plus := String,+", "times := String,*", "open_parenthesis := String,(","close_parenthesis := String,)"] productionset_arithmetic = strlist_to_production_set(arithmetic, start_symbol= "E") addition=["S ::= E","E ::= E plus F | F" ,"F ::= open_parenthesis E close_parenthesis | id", "id := String,123" , "plus := String,+", "open_parenthesis := String,(","close_parenthesis := String,)"] productionset_addition = strlist_to_production_set(addition) #tokenlist definition string1 = "S:a" string2 = "S:" string3 = "AB"
final3 = NonTerminalSymbol("exp") rule1 = Production([final1], (symbol1, symbol3, symbol5)) rule2 = Production([final2], (symbol2, symbol3, symbol4)) rule3 = Production([final3], [final1]) rule4 = Production([final3], [final2]) rulelist = (rule1, rule2, rule3, rule4, symbol1, symbol2, symbol3, symbol4, symbol5) productionset1 = BNFGrammar(final3, rulelist) #productionset2 definition symbola = TerminalSymbol(String("A")) symbolb = TerminalSymbol(String("B")) nonterminal = NonTerminalSymbol("res") rulea = Production((nonterminal, ), (symbola, NullSymbol(), symbolb)) productionset2 = BNFGrammar(nonterminal, (rulea, symbola, symbolb)) productionsetlr = strlist_to_production_set(leftrecursive) productionsetrr = strlist_to_production_set(rightrecursive) productionsetcr = strlist_to_production_set(centerrecursive) #arithmetic arithmetic = [ "E ::= E plus T | T", "T ::= T times F | F", "F ::= open_parenthesis E close_parenthesis | id", "id := String,123", "plus := String,+", "times := String,*", "open_parenthesis := String,(", "close_parenthesis := String,)" ] productionset_arithmetic = strlist_to_production_set(arithmetic, start_symbol="E") addition = [