def test_Concept(self): red = String("red") green = String("green") blue = String("blue") alphabet = Choice([red, green, blue], ascii_encoding) lexer = lexer_factory(alphabet, ascii_encoding) def concept_translator_fun(inputtokens): result = [] for token in inputtokens: x = str(token) if x == "red": result.append("color red") elif x == "green": result.append("color green") elif x == "blue": result.append("color blue") else: raise Exception("%s,%s" % (x, x.__class__.__name__)) return result ct = concept_translator_fun self.assertListEqual(ct(lexer("red")), ["color red"]) red_list = [ PositionToken(content=character, gd=ascii_encoding, left=i, right=i + 1) for i, character in enumerate("red") ] self.assertListEqual(ct(lexer(red_list)), ["color red"])
def testCheck(self): from pydsl.grammar.PEG import Sequence from pydsl.check import SequenceChecker sequence = Sequence((String("a"), String("b"), String("c"))) checker = SequenceChecker(sequence) self.assertTrue(checker.check("abc")) self.assertTrue(checker.check([x for x in "abc"])) self.assertFalse(checker.check("abd")) self.assertFalse(checker.check(""))
def testZeroOrMore(self): mygrammar = ZeroOrMore(String("a")) self.assertTrue(isinstance(mygrammar, Grammar)) self.assertEqual(mygrammar.first(), Choice([String("a")])) self.assertTrue(check(mygrammar, "a")) self.assertTrue(check(mygrammar, "aa")) self.assertTrue(check(mygrammar, "aaaa")) self.assertTrue(check(mygrammar, "")) self.assertFalse(check(mygrammar, "b"))
def testCheck(self): """Test checker instantiation and call""" from pydsl.grammar.definition import JsonSchema from pydsl.check import JsonSchemaChecker schema = { "type" : "object", "required":["foo"], "properties" : { "foo" : {"enum" : [1, 3]}, "bar" : {"format": "number_three"} #Ignored by jsonschema } } grammardef = JsonSchema(schema) checker = JsonSchemaChecker(grammardef) self.assertFalse(checker.check("a")) self.assertTrue(checker.check({"foo":1})) self.assertFalse(checker.check({"foo":2})) self.assertTrue(checker.check({"foo":3})) self.assertFalse(checker.check([1, {"foo" : 2, "bar" : {"baz" : [1]}}, "quux"])) self.assertRaises(Exception, checker.check, [1, {"foo" : 2, "bar" : {"baz" : [1]}}, "quux"], raise_exceptions=True) number_three = checker_factory(String("3")) fc = {"number_three":number_three} grammardef = JsonSchema(schema) checker = JsonSchemaChecker(grammardef, fc) # Adds a format checker self.assertFalse(checker.check({"foo" : 1, "bar" : "123456"})) self.assertTrue(checker.check({"foo" : 1, "bar" : "3"}))
def testDiffSimple(self): alphabet = Choice([String(x) for x in "abcde1"]) elem1 = "abcde" elem2 = "abcd1" from pydsl.diff import diff self.assertEqual(diff(alphabet, elem1, elem2)[0].a, 0) self.assertEqual(diff(alphabet, elem1, elem2)[0].b, 0) self.assertEqual(diff(alphabet, elem1, elem2)[0].size, 4)
def testCheck(self): """Test checker instantiation and call""" from pydsl.check import StringChecker grammarchecker = StringChecker(String("3")) self.assertTrue(grammarchecker("3")) self.assertTrue(grammarchecker(["3"])) self.assertTrue(grammarchecker(("3", ))) self.assertFalse(grammarchecker(''))
def __generateStringSymbol(rightside): head, tail = rightside.split(",", 1) if head != "String": raise TypeError content = tail if len(tail) > 2 and tail[1][0] == "'" and tail[1][-1] == "'": content = tail[1][1:-1] from pydsl.grammar.definition import String return TerminalSymbol(String(content))
def testSimpleChoiceLexer(self): a1 = Choice([String('a'), String('b'), String('c')]) from pydsl.lex import ChoiceLexer lexer = ChoiceLexer(a1) self.assertListEqual(lexer("abc"), [ Token("a", String('a')), Token("b", String("b")), Token("c", String("c")) ])
class TestGrammarDefinitionString(unittest.TestCase): def setUp(self): self.grammardef = String('abc') def testEnumerate(self): self.assertListEqual(['abc'], [x for x in self.grammardef.enum()]) def testFirst(self): self.assertEqual(self.grammardef.first, String('a')) def testMin(self): self.assertEqual(self.grammardef.minsize, 3) def testMax(self): self.assertEqual(self.grammardef.maxsize, 3) def testAlphabet(self): self.assertSetEqual(self.grammardef.alphabet, ascii_encoding)
def test_calculator_simple(self): grammar_def = [ "S ::= E", "E ::= number operator number", "number := Word,integer,max", "operator := String,+", ] from pydsl.file.BNF import strlist_to_production_set from pydsl.grammar import RegularExpression repository = {'integer': RegularExpression("^[0123456789]*$")} production_set = strlist_to_production_set(grammar_def, repository) rdp = LL1RecursiveDescentParser(production_set) parse_tree = rdp("1+2") def parse_tree_walker(tree): from pydsl.grammar.symbol import NonTerminalSymbol if tree.symbol == NonTerminalSymbol("S"): return parse_tree_walker(tree.childlist[0]) if tree.symbol == NonTerminalSymbol("E"): return int(str(tree.childlist[0].content)) + int( str(tree.childlist[2].content)) else: raise Exception result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 3) from pydsl.grammar.PEG import Choice from pydsl.grammar.definition import String, RegularExpression from pydsl.encoding import ascii_encoding math_alphabet = Choice( [RegularExpression("^[0123456789]*$"), Choice([String('+')])]) from pydsl.lex import lex tokens = [x for x in lex(math_alphabet, ascii_encoding, "11+2")] parse_tree = rdp(tokens) result = parse_tree_walker(parse_tree[0]) self.assertEqual(result, 13)
def testSecondLevelGrammar(self): a = String("a") b = String("b") c = String("c") x = String("x") y = String("y") z = String("z") first_level = Choice([a, b, c]) first_levelb = Choice([x, y, z]) second_level = Sequence([a, b], base_alphabet=first_level) from pydsl.check import checker_factory checker = checker_factory(second_level) self.assertTrue(checker([a, b])) second_level_alphabet = Choice([first_level, first_levelb]) lexer = lexer_factory(second_level_alphabet, base=first_level + first_levelb) self.assertListEqual( lexer("ab"), [Token("a", first_level), Token("b", first_level)])
def testNot(self): mygrammar = Not(String("a")) self.assertTrue(isinstance(mygrammar, Not))
def testFirst(self): re1 = RegularExpression(re.compile('^a$')) self.assertEqual(len(re1.first),1) from pydsl.grammar.definition import String self.assertIn(String('a'), re1.first)
def setUp(self): self.grammardef = String('abc')
def testFirstLookup(self): from pydsl.grammar.symbol import NonTerminalSymbol, TerminalSymbol from pydsl.grammar.PEG import Choice self.assertEqual( self.grammardef.first_lookup(NonTerminalSymbol("exp")), Choice([String("S")]))
def testFirst(self): self.assertEqual(self.grammardef.first, String('a'))
def testChoice(self): mygrammar = Choice((String("a"), String("b"))) self.assertTrue(check(mygrammar, "a")) self.assertTrue(check(mygrammar, "b")) self.assertFalse(check(mygrammar, "c"))
"""BNF grammars for testing""" from pydsl.grammar.symbol import TerminalSymbol, NonTerminalSymbol, NullSymbol from pydsl.grammar.BNF import Production, BNFGrammar from pydsl.file.BNF import strlist_to_production_set from pydsl.file.python import load_python_file from pydsl.grammar.definition import String, RegularExpression leftrecursive = ["S ::= E", "E ::= E dot | dot", "dot := String,."] rightrecursive = ["S ::= E", "E ::= dot E | dot", "dot := String,."] centerrecursive = ["S ::= E", "E ::= dot E dot | dot", "dot := String,."] #productionset0 definition symbol1 = TerminalSymbol(String("S")) symbol2 = TerminalSymbol(String("R")) final1 = NonTerminalSymbol("exp") rule1 = Production([final1], (symbol1, symbol2)) productionset0 = BNFGrammar(final1, (rule1, symbol1, symbol2)) p0good = "SR" p0bad = "RS" #productionset1 definition symbol1 = TerminalSymbol(String("S")) symbol2 = TerminalSymbol(String("R")) symbol3 = TerminalSymbol(String(":")) symbol4 = TerminalSymbol(RegularExpression("^[0123456789]*$")) symbol5 = TerminalSymbol(load_python_file('pydsl/contrib/grammar/cstring.py')) final1 = NonTerminalSymbol("storeexp") final2 = NonTerminalSymbol("retrieveexp") final3 = NonTerminalSymbol("exp")
def testBasic(self): self.assertTrue(equal(String('a'), 'a', 'a')) self.assertRaises(ValueError, equal, String('a'), 'b', 'a')
def testFirst(self): self.assertSetEqual(self.grammardef.first, set([String("S")]))
def testSequence(self): mygrammar = Sequence((String("a"), String("b"))) self.assertTrue(isinstance(mygrammar, Grammar))
def testAlphabet(self): self.assertEqual(self.grammardef.alphabet, set([String(x) for x in ["S", "R"]]))
def test_simple(self): alphabet = Choice([String('a')]) base = Choice([String('a')]) graph = graph_from_alphabet(alphabet, base) self.assertEqual(len(graph.node), 2)