def test_Concept(self):
red = String("red")
green = String("green")
blue = String("blue")
alphabet = Choice([red, green, blue], ascii_encoding)
lexer = lexer_factory(alphabet, ascii_encoding)
def concept_translator_fun(inputtokens):
result = []
for token in inputtokens:
x = str(token)
if x == "red":
result.append("color red")
elif x == "green":
result.append("color green")
elif x == "blue":
result.append("color blue")
else:
raise Exception("%s,%s" % (x, x.__class__.__name__))
return result
ct = concept_translator_fun
self.assertListEqual(ct(lexer("red")), ["color red"])
red_list = [
PositionToken(content=character,
gd=ascii_encoding,
left=i,
right=i + 1) for i, character in enumerate("red")
]
self.assertListEqual(ct(lexer(red_list)), ["color red"])
def testCheck(self):
from pydsl.grammar.PEG import Sequence
from pydsl.check import SequenceChecker
sequence = Sequence((String("a"), String("b"), String("c")))
checker = SequenceChecker(sequence)
self.assertTrue(checker.check("abc"))
self.assertTrue(checker.check([x for x in "abc"]))
self.assertFalse(checker.check("abd"))
self.assertFalse(checker.check(""))
def testZeroOrMore(self):
mygrammar = ZeroOrMore(String("a"))
self.assertTrue(isinstance(mygrammar, Grammar))
self.assertEqual(mygrammar.first(), Choice([String("a")]))
self.assertTrue(check(mygrammar, "a"))
self.assertTrue(check(mygrammar, "aa"))
self.assertTrue(check(mygrammar, "aaaa"))
self.assertTrue(check(mygrammar, ""))
self.assertFalse(check(mygrammar, "b"))
def testCheck(self):
"""Test checker instantiation and call"""
from pydsl.grammar.definition import JsonSchema
from pydsl.check import JsonSchemaChecker
schema = {
"type" : "object",
"required":["foo"],
"properties" : {
"foo" : {"enum" : [1, 3]},
"bar" : {"format": "number_three"} #Ignored by jsonschema
}
}
grammardef = JsonSchema(schema)
checker = JsonSchemaChecker(grammardef)
self.assertFalse(checker.check("a"))
self.assertTrue(checker.check({"foo":1}))
self.assertFalse(checker.check({"foo":2}))
self.assertTrue(checker.check({"foo":3}))
self.assertFalse(checker.check([1, {"foo" : 2, "bar" : {"baz" : [1]}}, "quux"]))
self.assertRaises(Exception, checker.check, [1, {"foo" : 2, "bar" : {"baz" : [1]}}, "quux"], raise_exceptions=True)
number_three = checker_factory(String("3"))
fc = {"number_three":number_three}
grammardef = JsonSchema(schema)
checker = JsonSchemaChecker(grammardef, fc) # Adds a format checker
self.assertFalse(checker.check({"foo" : 1, "bar" : "123456"}))
self.assertTrue(checker.check({"foo" : 1, "bar" : "3"}))
def testDiffSimple(self):
alphabet = Choice([String(x) for x in "abcde1"])
elem1 = "abcde"
elem2 = "abcd1"
from pydsl.diff import diff
self.assertEqual(diff(alphabet, elem1, elem2)[0].a, 0)
self.assertEqual(diff(alphabet, elem1, elem2)[0].b, 0)
self.assertEqual(diff(alphabet, elem1, elem2)[0].size, 4)
def testCheck(self):
"""Test checker instantiation and call"""
from pydsl.check import StringChecker
grammarchecker = StringChecker(String("3"))
self.assertTrue(grammarchecker("3"))
self.assertTrue(grammarchecker(["3"]))
self.assertTrue(grammarchecker(("3", )))
self.assertFalse(grammarchecker(''))
def __generateStringSymbol(rightside):
head, tail = rightside.split(",", 1)
if head != "String":
raise TypeError
content = tail
if len(tail) > 2 and tail[1][0] == "'" and tail[1][-1] == "'":
content = tail[1][1:-1]
from pydsl.grammar.definition import String
return TerminalSymbol(String(content))
def testSimpleChoiceLexer(self):
a1 = Choice([String('a'), String('b'), String('c')])
from pydsl.lex import ChoiceLexer
lexer = ChoiceLexer(a1)
self.assertListEqual(lexer("abc"), [
Token("a", String('a')),
Token("b", String("b")),
Token("c", String("c"))
])
class TestGrammarDefinitionString(unittest.TestCase):
def setUp(self):
self.grammardef = String('abc')
def testEnumerate(self):
self.assertListEqual(['abc'], [x for x in self.grammardef.enum()])
def testFirst(self):
self.assertEqual(self.grammardef.first, String('a'))
def testMin(self):
self.assertEqual(self.grammardef.minsize, 3)
def testMax(self):
self.assertEqual(self.grammardef.maxsize, 3)
def testAlphabet(self):
self.assertSetEqual(self.grammardef.alphabet, ascii_encoding)
class TestGrammarDefinitionString(unittest.TestCase):
def setUp(self):
self.grammardef = String('abc')
def testEnumerate(self):
self.assertListEqual(['abc'], [x for x in self.grammardef.enum()])
def testFirst(self):
self.assertEqual(self.grammardef.first, String('a'))
def testMin(self):
self.assertEqual(self.grammardef.minsize, 3)
def testMax(self):
self.assertEqual(self.grammardef.maxsize, 3)
def testAlphabet(self):
self.assertSetEqual(self.grammardef.alphabet, ascii_encoding)
def test_calculator_simple(self):
grammar_def = [
"S ::= E",
"E ::= number operator number",
"number := Word,integer,max",
"operator := String,+",
]
from pydsl.file.BNF import strlist_to_production_set
from pydsl.grammar import RegularExpression
repository = {'integer': RegularExpression("^[0123456789]*$")}
production_set = strlist_to_production_set(grammar_def, repository)
rdp = LL1RecursiveDescentParser(production_set)
parse_tree = rdp("1+2")
def parse_tree_walker(tree):
from pydsl.grammar.symbol import NonTerminalSymbol
if tree.symbol == NonTerminalSymbol("S"):
return parse_tree_walker(tree.childlist[0])
if tree.symbol == NonTerminalSymbol("E"):
return int(str(tree.childlist[0].content)) + int(
str(tree.childlist[2].content))
else:
raise Exception
result = parse_tree_walker(parse_tree[0])
self.assertEqual(result, 3)
from pydsl.grammar.PEG import Choice
from pydsl.grammar.definition import String, RegularExpression
from pydsl.encoding import ascii_encoding
math_alphabet = Choice(
[RegularExpression("^[0123456789]*$"),
Choice([String('+')])])
from pydsl.lex import lex
tokens = [x for x in lex(math_alphabet, ascii_encoding, "11+2")]
parse_tree = rdp(tokens)
result = parse_tree_walker(parse_tree[0])
self.assertEqual(result, 13)
def testSecondLevelGrammar(self):
a = String("a")
b = String("b")
c = String("c")
x = String("x")
y = String("y")
z = String("z")
first_level = Choice([a, b, c])
first_levelb = Choice([x, y, z])
second_level = Sequence([a, b], base_alphabet=first_level)
from pydsl.check import checker_factory
checker = checker_factory(second_level)
self.assertTrue(checker([a, b]))
second_level_alphabet = Choice([first_level, first_levelb])
lexer = lexer_factory(second_level_alphabet,
base=first_level + first_levelb)
self.assertListEqual(
lexer("ab"), [Token("a", first_level),
Token("b", first_level)])
def testNot(self):
mygrammar = Not(String("a"))
self.assertTrue(isinstance(mygrammar, Not))
def testFirst(self):
re1 = RegularExpression(re.compile('^a$'))
self.assertEqual(len(re1.first),1)
from pydsl.grammar.definition import String
self.assertIn(String('a'), re1.first)
def setUp(self):
self.grammardef = String('abc')
def setUp(self):
self.grammardef = String('abc')
def testFirstLookup(self):
from pydsl.grammar.symbol import NonTerminalSymbol, TerminalSymbol
from pydsl.grammar.PEG import Choice
self.assertEqual(
self.grammardef.first_lookup(NonTerminalSymbol("exp")),
Choice([String("S")]))
def testFirst(self):
self.assertEqual(self.grammardef.first, String('a'))
def testChoice(self):
mygrammar = Choice((String("a"), String("b")))
self.assertTrue(check(mygrammar, "a"))
self.assertTrue(check(mygrammar, "b"))
self.assertFalse(check(mygrammar, "c"))
"""BNF grammars for testing"""
from pydsl.grammar.symbol import TerminalSymbol, NonTerminalSymbol, NullSymbol
from pydsl.grammar.BNF import Production, BNFGrammar
from pydsl.file.BNF import strlist_to_production_set
from pydsl.file.python import load_python_file
from pydsl.grammar.definition import String, RegularExpression
leftrecursive = ["S ::= E", "E ::= E dot | dot", "dot := String,."]
rightrecursive = ["S ::= E", "E ::= dot E | dot", "dot := String,."]
centerrecursive = ["S ::= E", "E ::= dot E dot | dot", "dot := String,."]
#productionset0 definition
symbol1 = TerminalSymbol(String("S"))
symbol2 = TerminalSymbol(String("R"))
final1 = NonTerminalSymbol("exp")
rule1 = Production([final1], (symbol1, symbol2))
productionset0 = BNFGrammar(final1, (rule1, symbol1, symbol2))
p0good = "SR"
p0bad = "RS"
#productionset1 definition
symbol1 = TerminalSymbol(String("S"))
symbol2 = TerminalSymbol(String("R"))
symbol3 = TerminalSymbol(String(":"))
symbol4 = TerminalSymbol(RegularExpression("^[0123456789]*$"))
symbol5 = TerminalSymbol(load_python_file('pydsl/contrib/grammar/cstring.py'))
final1 = NonTerminalSymbol("storeexp")
final2 = NonTerminalSymbol("retrieveexp")
final3 = NonTerminalSymbol("exp")
def testBasic(self):
self.assertTrue(equal(String('a'), 'a', 'a'))
self.assertRaises(ValueError, equal, String('a'), 'b', 'a')
def testFirst(self):
self.assertSetEqual(self.grammardef.first, set([String("S")]))
def testSequence(self):
mygrammar = Sequence((String("a"), String("b")))
self.assertTrue(isinstance(mygrammar, Grammar))
def testAlphabet(self):
self.assertEqual(self.grammardef.alphabet,
set([String(x) for x in ["S", "R"]]))
def test_simple(self):
alphabet = Choice([String('a')])
base = Choice([String('a')])
graph = graph_from_alphabet(alphabet, base)
self.assertEqual(len(graph.node), 2)
