def testFirstLookup(self): from pydsl.Grammar.Symbol import NonTerminalSymbol, TerminalSymbol from pydsl.Grammar.PEG import Choice self.grammardef.first_lookup(NonTerminalSymbol("exp"))[0] self.assertEqual( self.grammardef.first_lookup(NonTerminalSymbol("exp")), Choice([String("S")]))
def test_Concept(self): red = String("red") green = String("green") blue = String("blue") alphabet = Choice([red, green, blue]) lexer = lexer_factory(alphabet) def concept_translator_fun(inputtokens): result = [] for x, _ in inputtokens: if x == "red" or x == ["r", "e", "d"]: result.append("color red") elif x == "green" or x == ["g", "r", "e", "e", "n"]: result.append("color green") elif x == "blue" or x == ["b", "l", "u", "e"]: result.append("color blue") else: raise Exception("%s,%s" % (x, x.__class__.__name__)) return result ct = concept_translator_fun self.assertListEqual(ct(lexer("red")), ["color red"]) self.assertListEqual(ct(lexer([x for x in "red"])), ["color red"])
def first(self): """Returns the a grammar definition that includes all first elements of this grammar""" #TODO result = [] for x in self.first_lookup(self.initialsymbol): result += x.first() if len(result) == 1: return result[0] return Choice(result)
def testSecondLevelGrammar(self): a = String("a") b = String("b") c = String("c") x = String("x") y = String("y") z = String("z") first_level = Choice([a, b, c]) first_levelb = Choice([x, y, z]) second_level = Sequence([a, b], base_alphabet=first_level) from pydsl.Check import checker_factory checker = checker_factory(second_level) self.assertTrue(checker([a, b])) second_level_alphabet = Choice([first_level, first_levelb]) lexer = lexer_factory(second_level_alphabet, base=first_level + first_levelb) self.assertListEqual(lexer("ab"), [("a", first_level), ("b", first_level)])
def testCheck(self): from pydsl.Check import ChoiceChecker from pydsl.Grammar.PEG import Choice from pydsl.Grammar import RegularExpression a = Choice([RegularExpression('^[0123456789]*$')]) checker = ChoiceChecker(a) self.assertTrue(checker.check([x for x in '1234'])) self.assertTrue(checker.check('1234')) self.assertFalse(checker.check('abc')) self.assertFalse(checker.check(''))
def testChoices(self): gd = Choice([String('a'), String('b'), String('c')]) self.assertListEqual(extract_alphabet(gd, 'axbycz'), [ PositionToken('a', None, 0, 1), PositionToken('b', None, 2, 3), PositionToken('c', None, 4, 5) ]) self.assertListEqual(extract_alphabet(gd, 'xyzabcxyz'), [PositionToken('abc', None, 3, 6)]) self.assertListEqual(extract_alphabet(gd, 'abcxyz'), [PositionToken('abc', None, 0, 3)]) self.assertListEqual( extract_alphabet(gd, [Token(x, None) for x in 'abcxyz']), [PositionToken(['a', 'b', 'c'], None, 0, 3)]) self.assertListEqual(extract_alphabet(gd, 'abc'), [PositionToken('abc', None, 0, 3)]) self.assertListEqual(extract_alphabet(gd, ''), [])
def first_lookup(self, symbol, size=1): """ Returns a Grammar Definition with the first n terminal symbols produced by the input symbol """ if isinstance(symbol, (TerminalSymbol, NullSymbol)): return [symbol.gd] result = [] for production in self.productions: if production.leftside[0] != symbol: continue for right_symbol in production.rightside: if right_symbol == symbol: #Avoids infinite recursion break current_symbol_first = self.first_lookup(right_symbol, size) result += current_symbol_first if NullSymbol not in current_symbol_first: break # This element doesn't have Null in its first set so there is no need to continue if not result: raise KeyError("Symbol doesn't exist in this grammar") from pydsl.Grammar.PEG import Choice return Choice(result)
def test_calculator_simple(self): grammar_def = [ "S ::= E", "E ::= number operator number", "number := Word,integer,max", "operator := String,+", ] from pydsl.File.BNF import strlist_to_production_set from pydsl.Grammar import RegularExpression repository = {'integer': RegularExpression("^[0123456789]*$")} production_set = strlist_to_production_set(grammar_def, repository) rdp = LL1RecursiveDescentParser(production_set) parse_tree = rdp("1+2") def parse_tree_walker(tree): from pydsl.Grammar.Symbol import NonTerminalSymbol if tree.symbol == NonTerminalSymbol("S"): return parse_tree_walker(tree.childlist[0]) if tree.symbol == NonTerminalSymbol("E"): return int(str(tree.childlist[0].content)) + int( str(tree.childlist[2].content)) else: raise Exception result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 3) from pydsl.Grammar.PEG import Choice from pydsl.Grammar.Definition import String, RegularExpression math_alphabet = Choice( [RegularExpression("^[0123456789]*$"), String('+')]) ascii_encoding = Encoding("ascii") from pydsl.Lex import lex tokens = [x[0] for x in lex(math_alphabet, ascii_encoding, "11+2")] parse_tree = rdp(tokens) result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 13)
def testChoice(self): mygrammar = Choice((String("a"), String("b"))) from pydsl.Check import check self.assertTrue(check(mygrammar, "a")) self.assertTrue(check(mygrammar, "b")) self.assertFalse(check(mygrammar, "c"))
def testSimpleChoiceLexer(self): a1 = Choice([String('a'), String('b'), String('c')]) from pydsl.Lex import ChoiceLexer lexer = ChoiceLexer(a1) self.assertListEqual(lexer("abc"), [("a", String('a'))])