def test_Concept(self):
red = String("red")
green = String("green")
blue = String("blue")
alphabet = Choice([red, green, blue])
lexer = lexer_factory(alphabet)
def concept_translator_fun(inputtokens):
result = []
for x, _ in inputtokens:
if x == "red" or x == ["r", "e", "d"]:
result.append("color red")
elif x == "green" or x == ["g", "r", "e", "e", "n"]:
result.append("color green")
elif x == "blue" or x == ["b", "l", "u", "e"]:
result.append("color blue")
else:
raise Exception("%s,%s" % (x, x.__class__.__name__))
return result
ct = concept_translator_fun
self.assertListEqual(ct(lexer("red")), ["color red"])
self.assertListEqual(ct(lexer([x for x in "red"])), ["color red"])
def testCheck(self):
from pydsl.Grammar.PEG import Sequence
from pydsl.Check import SequenceChecker
sequence = Sequence((String("a"), String("b"), String("c")))
checker = SequenceChecker(sequence)
self.assertTrue(checker.check("abc"))
self.assertTrue(checker.check([x for x in "abc"]))
self.assertFalse(checker.check("abd"))
self.assertFalse(checker.check(""))
def testFirstLookup(self):
from pydsl.Grammar.Symbol import NonTerminalSymbol, TerminalSymbol
from pydsl.Grammar.PEG import Choice
self.grammardef.first_lookup(NonTerminalSymbol("exp"))[0]
self.assertEqual(
self.grammardef.first_lookup(NonTerminalSymbol("exp")),
Choice([String("S")]))
def __generateStringSymbol(rightside):
head, tail = rightside.split(",", 1)
if head != "String":
raise TypeError
content = tail
if len(tail) > 2 and tail[1][0] == "'" and tail[1][-1] == "'":
content = tail[1][1:-1]
from pydsl.Grammar.Definition import String
return TerminalSymbol(String(content))
def testCheck(self):
"""Test checker instantiation and call"""
from pydsl.Check import StringChecker
grammarchecker = StringChecker(String("string123"))
self.assertTrue(grammarchecker("string123"))
self.assertTrue(grammarchecker(["string123"]))
self.assertTrue(grammarchecker(("string123", )))
list_version = ["s", "t", "r", "i", "n", "g", "1", "2", "3"]
self.assertTrue(
grammarchecker((
"s",
"t",
"r",
"i",
"n",
"g",
"1",
"2",
"3",
)))
self.assertTrue(grammarchecker(list_version))
self.assertTrue(grammarchecker([String(x) for x in list_version]))
self.assertTrue(grammarchecker([x for x in list_version]))
self.assertFalse(grammarchecker(''))
def testSecondLevelGrammar(self):
a = String("a")
b = String("b")
c = String("c")
x = String("x")
y = String("y")
z = String("z")
first_level = Choice([a, b, c])
first_levelb = Choice([x, y, z])
second_level = Sequence([a, b], base_alphabet=first_level)
from pydsl.Check import checker_factory
checker = checker_factory(second_level)
self.assertTrue(checker([a, b]))
second_level_alphabet = Choice([first_level, first_levelb])
lexer = lexer_factory(second_level_alphabet,
base=first_level + first_levelb)
self.assertListEqual(lexer("ab"), [("a", first_level),
("b", first_level)])
def test_calculator_simple(self):
grammar_def = [
"S ::= E",
"E ::= number operator number",
"number := Word,integer,max",
"operator := String,+",
]
from pydsl.File.BNF import strlist_to_production_set
from pydsl.Grammar import RegularExpression
repository = {'integer': RegularExpression("^[0123456789]*$")}
production_set = strlist_to_production_set(grammar_def, repository)
rdp = LL1RecursiveDescentParser(production_set)
parse_tree = rdp("1+2")
def parse_tree_walker(tree):
from pydsl.Grammar.Symbol import NonTerminalSymbol
if tree.symbol == NonTerminalSymbol("S"):
return parse_tree_walker(tree.childlist[0])
if tree.symbol == NonTerminalSymbol("E"):
return int(str(tree.childlist[0].content)) + int(
str(tree.childlist[2].content))
else:
raise Exception
result = parse_tree_walker(parse_tree[0])
self.assertEqual(result, 3)
from pydsl.Grammar.PEG import Choice
from pydsl.Grammar.Definition import String, RegularExpression
math_alphabet = Choice(
[RegularExpression("^[0123456789]*$"),
String('+')])
ascii_encoding = Encoding("ascii")
from pydsl.Lex import lex
tokens = [x[0] for x in lex(math_alphabet, ascii_encoding, "11+2")]
parse_tree = rdp(tokens)
result = parse_tree_walker(parse_tree[0])
self.assertEqual(result, 13)
def testSequence(self):
mygrammar = Sequence((String("a"), String("b")))
self.assertTrue(isinstance(mygrammar, Grammar))
def testNot(self):
mygrammar = Not(String("a"))
self.assertTrue(isinstance(mygrammar, Grammar))
def testChoice(self):
mygrammar = Choice((String("a"), String("b")))
from pydsl.Check import check
self.assertTrue(check(mygrammar, "a"))
self.assertTrue(check(mygrammar, "b"))
self.assertFalse(check(mygrammar, "c"))
def testMany(self):
mygrammar = Many(String("a"))
self.assertTrue(isinstance(mygrammar, Grammar))
def testSimpleChoiceLexer(self):
a1 = Choice([String('a'), String('b'), String('c')])
from pydsl.Lex import ChoiceLexer
lexer = ChoiceLexer(a1)
self.assertListEqual(lexer("abc"), [("a", String('a'))])
"""BNF grammars for testing"""
from pydsl.Grammar.Symbol import TerminalSymbol, NonTerminalSymbol, NullSymbol
from pydsl.Grammar.BNF import Production, BNFGrammar
from pydsl.File.BNF import strlist_to_production_set
from pydsl.File.Python import load_python_file
from pydsl.Grammar.Definition import String, RegularExpression
br = "max"
leftrecursive=["S ::= E","E ::= E dot | dot","dot := String,."]
rightrecursive=["S ::= E","E ::= dot E | dot","dot := String,."]
centerrecursive=["S ::= E","E ::= dot E dot | dot","dot := String,."]
#productionset0 definition
symbol1 = TerminalSymbol(String("S"))
symbol2 = TerminalSymbol(String("R"))
final1 = NonTerminalSymbol("exp")
rule1 = Production([final1], (symbol1, symbol2))
productionset0 = BNFGrammar(final1, (rule1,symbol1,symbol2))
p0good = "SR"
p0bad = "SRL"
#productionset1 definition
symbol1 = TerminalSymbol(String("S"))
symbol2 = TerminalSymbol(String("R"))
symbol3 = TerminalSymbol(String(":"))
symbol4 = TerminalSymbol(RegularExpression("^[0123456789]*$"), None, br)
symbol5 = TerminalSymbol(load_python_file('pydsl/contrib/grammar/cstring.py'), None, br)
final1 = NonTerminalSymbol("storeexp")
def testFirst(self):
re1 = RegularExpression(re.compile('^a$'))
self.assertEqual(len(re1.first()), 1)
from pydsl.Grammar.Definition import String
self.assertEqual(re1.first()[0], String('a'))
def testAlphabet(self):
self.assertListEqual(list(self.grammardef.alphabet),
[String(x) for x in ["S", "R"]])
def testFirst(self):
self.assertEqual(self.grammardef.first, String("S"))