def testFirstLookup(self):
from pydsl.Grammar.Symbol import NonTerminalSymbol, TerminalSymbol
from pydsl.Grammar.PEG import Choice
self.grammardef.first_lookup(NonTerminalSymbol("exp"))[0]
self.assertEqual(
self.grammardef.first_lookup(NonTerminalSymbol("exp")),
Choice([String("S")]))
def test_Concept(self):
red = String("red")
green = String("green")
blue = String("blue")
alphabet = Choice([red, green, blue])
lexer = lexer_factory(alphabet)
def concept_translator_fun(inputtokens):
result = []
for x, _ in inputtokens:
if x == "red" or x == ["r", "e", "d"]:
result.append("color red")
elif x == "green" or x == ["g", "r", "e", "e", "n"]:
result.append("color green")
elif x == "blue" or x == ["b", "l", "u", "e"]:
result.append("color blue")
else:
raise Exception("%s,%s" % (x, x.__class__.__name__))
return result
ct = concept_translator_fun
self.assertListEqual(ct(lexer("red")), ["color red"])
self.assertListEqual(ct(lexer([x for x in "red"])), ["color red"])
def first(self):
"""Returns the a grammar definition that includes all first elements of this grammar""" #TODO
result = []
for x in self.first_lookup(self.initialsymbol):
result += x.first()
if len(result) == 1:
return result[0]
return Choice(result)
def testSecondLevelGrammar(self):
a = String("a")
b = String("b")
c = String("c")
x = String("x")
y = String("y")
z = String("z")
first_level = Choice([a, b, c])
first_levelb = Choice([x, y, z])
second_level = Sequence([a, b], base_alphabet=first_level)
from pydsl.Check import checker_factory
checker = checker_factory(second_level)
self.assertTrue(checker([a, b]))
second_level_alphabet = Choice([first_level, first_levelb])
lexer = lexer_factory(second_level_alphabet,
base=first_level + first_levelb)
self.assertListEqual(lexer("ab"), [("a", first_level),
("b", first_level)])
def testCheck(self):
from pydsl.Check import ChoiceChecker
from pydsl.Grammar.PEG import Choice
from pydsl.Grammar import RegularExpression
a = Choice([RegularExpression('^[0123456789]*$')])
checker = ChoiceChecker(a)
self.assertTrue(checker.check([x for x in '1234']))
self.assertTrue(checker.check('1234'))
self.assertFalse(checker.check('abc'))
self.assertFalse(checker.check(''))
def testChoices(self):
gd = Choice([String('a'), String('b'), String('c')])
self.assertListEqual(extract_alphabet(gd, 'axbycz'), [
PositionToken('a', None, 0, 1),
PositionToken('b', None, 2, 3),
PositionToken('c', None, 4, 5)
])
self.assertListEqual(extract_alphabet(gd, 'xyzabcxyz'),
[PositionToken('abc', None, 3, 6)])
self.assertListEqual(extract_alphabet(gd, 'abcxyz'),
[PositionToken('abc', None, 0, 3)])
self.assertListEqual(
extract_alphabet(gd, [Token(x, None) for x in 'abcxyz']),
[PositionToken(['a', 'b', 'c'], None, 0, 3)])
self.assertListEqual(extract_alphabet(gd, 'abc'),
[PositionToken('abc', None, 0, 3)])
self.assertListEqual(extract_alphabet(gd, ''), [])
def first_lookup(self, symbol, size=1):
"""
Returns a Grammar Definition with the first n terminal symbols
produced by the input symbol
"""
if isinstance(symbol, (TerminalSymbol, NullSymbol)):
return [symbol.gd]
result = []
for production in self.productions:
if production.leftside[0] != symbol:
continue
for right_symbol in production.rightside:
if right_symbol == symbol: #Avoids infinite recursion
break
current_symbol_first = self.first_lookup(right_symbol, size)
result += current_symbol_first
if NullSymbol not in current_symbol_first:
break # This element doesn't have Null in its first set so there is no need to continue
if not result:
raise KeyError("Symbol doesn't exist in this grammar")
from pydsl.Grammar.PEG import Choice
return Choice(result)
def test_calculator_simple(self):
grammar_def = [
"S ::= E",
"E ::= number operator number",
"number := Word,integer,max",
"operator := String,+",
]
from pydsl.File.BNF import strlist_to_production_set
from pydsl.Grammar import RegularExpression
repository = {'integer': RegularExpression("^[0123456789]*$")}
production_set = strlist_to_production_set(grammar_def, repository)
rdp = LL1RecursiveDescentParser(production_set)
parse_tree = rdp("1+2")
def parse_tree_walker(tree):
from pydsl.Grammar.Symbol import NonTerminalSymbol
if tree.symbol == NonTerminalSymbol("S"):
return parse_tree_walker(tree.childlist[0])
if tree.symbol == NonTerminalSymbol("E"):
return int(str(tree.childlist[0].content)) + int(
str(tree.childlist[2].content))
else:
raise Exception
result = parse_tree_walker(parse_tree[0])
self.assertEqual(result, 3)
from pydsl.Grammar.PEG import Choice
from pydsl.Grammar.Definition import String, RegularExpression
math_alphabet = Choice(
[RegularExpression("^[0123456789]*$"),
String('+')])
ascii_encoding = Encoding("ascii")
from pydsl.Lex import lex
tokens = [x[0] for x in lex(math_alphabet, ascii_encoding, "11+2")]
parse_tree = rdp(tokens)
result = parse_tree_walker(parse_tree[0])
self.assertEqual(result, 13)
def testChoice(self):
mygrammar = Choice((String("a"), String("b")))
from pydsl.Check import check
self.assertTrue(check(mygrammar, "a"))
self.assertTrue(check(mygrammar, "b"))
self.assertFalse(check(mygrammar, "c"))
def testSimpleChoiceLexer(self):
a1 = Choice([String('a'), String('b'), String('c')])
from pydsl.Lex import ChoiceLexer
lexer = ChoiceLexer(a1)
self.assertListEqual(lexer("abc"), [("a", String('a'))])