예제 #1
0
 def testLogicalExpression(self):
     repository = {'TrueFalse':load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")}
     productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/LogicalExpression.bnf", repository)
     parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
     tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "True&&False")]
     result = parser.get_trees(tokens)
     self.assertTrue(result)
     result = parser.get_trees("True&|False")
     self.assertFalse(result)
예제 #2
0
 def testLogicalExp(self):
     repository = {'TrueFalse':load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")}
     productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/LogicalExpression.bnf", repository)
     parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
     tokens = [x for x in lex(repository['TrueFalse'].alphabet, ascii_encoding, self.tokelist5)]
     self.assertEqual(len(tokens), 1)
     #tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, Encoding('ascii'), tokens)] #FIXME
     tokens = [Token('True', repository['TrueFalse'])]
     result = parser.get_trees(tokens)
     self.assertTrue(result)
예제 #3
0
 def testLogLine(self):
     repository = {
         'space':
         String(' '),
         'integer':
         RegularExpression("^[0123456789]*$"),
         'ipv4':
         RegularExpression(
             "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"),
         'characters':
         RegularExpression("^[A-z]+$")
     }
     grammar = load_bnf_file("pydsl/contrib/grammar/logline.bnf",
                             repository)
     checker = checker_factory(grammar)
     original_string = "1.2.3.4 - - [1/1/2003:11:11:11 +2] \"GET\" 1 1 \"referer\" \"useragent\""
     tokenized = lex(grammar.alphabet,
                     ascii_encoding,
                     original_string,
                     force_lexer="general")
     self.assertTrue(checker.check(tokenized))
     self.assertFalse(
         checker.check(
             "1.2.3.4 - - [1/1/2003:11:11:11 +2] \"GOT\" 1 1 \"referer\" \"useragent\""
         ))
예제 #4
0
 def testLogicalExpression(self):
     repository = {
         'TrueFalse': load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")
     }
     productionrulesetlogical = load_bnf_file(
         "pydsl/contrib/grammar/LogicalExpression.bnf", repository)
     parser = BacktracingErrorRecursiveDescentParser(
         productionrulesetlogical)
     tokens = [
         x[0] for x in lex(productionrulesetlogical.alphabet,
                           ascii_encoding, "True&&False")
     ]
     result = parser.get_trees(tokens)
     self.assertTrue(result)
     result = parser.get_trees("True&|False")
     self.assertFalse(result)
예제 #5
0
 def testOverlappingLexing(self):
     integer = RegularExpression("^[0123456789]*$")
     date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
     mydef = Alphabet([integer,date])
     lexer = lexer_factory(mydef)
     self.assertListEqual(lexer("123411/11/2001"), [("1234", integer),("11/11/2001", date)])
     self.assertListEqual(lexer([x for x in "123411/11/2001"]), [("1234", integer),("11/11/2001", date)])
예제 #6
0
 def testLogicalExp(self):
     repository = {
         'TrueFalse': load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")
     }
     productionrulesetlogical = load_bnf_file(
         "pydsl/contrib/grammar/LogicalExpression.bnf", repository)
     parser = BacktracingErrorRecursiveDescentParser(
         productionrulesetlogical)
     tokens = [
         x for x in lex(repository['TrueFalse'].alphabet, ascii_encoding,
                        self.tokelist5)
     ]
     self.assertEqual(len(tokens), 1)
     #tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, Encoding('ascii'), tokens)] #FIXME
     tokens = [Token('True', repository['TrueFalse'])]
     result = parser.get_trees(tokens)
     self.assertTrue(result)
예제 #7
0
파일: test_Lexer.py 프로젝트: nesaro/pydsl
 def testSimpleLexing(self):
     """Test checker instantiation and call"""
     integer = RegularExpression("^[0123456789]*$")
     date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
     mydef = Choice([integer, date])
     lexer = lexer_factory(mydef, ascii_encoding)
     self.assertListEqual(lexer("1234"), [Token("1234", integer)])
     self.assertListEqual(lexer([Token(x, ascii_encoding) for x in "1234"]), [Token("1234", integer)])
예제 #8
0
 def testSimpleLexing(self):
     """Test checker instantiation and call"""
     integer = RegularExpression("^[0123456789]*$")
     date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
     mydef = Alphabet([integer,date])
     lexer = lexer_factory(mydef)
     self.assertListEqual(lexer("1234"), [(["1","2","3","4"], integer)])
     self.assertListEqual(lexer([x for x in "1234"]), [(["1","2","3","4"], integer)])
예제 #9
0
 def testFileLoader(self):
     repository = {
         'integer': RegularExpression("^[0123456789]*$"),
         'DayOfMonth':
         load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
     }
     self.assertTrue(
         load_bnf_file("pydsl/contrib/grammar/Date.bnf", repository))
예제 #10
0
 def setUp(self):
     self.integer = RegularExpression("^[0123456789]*$")
     self.date = load_bnf_file(
         "pydsl/contrib/grammar/Date.bnf", {
             'integer':
             self.integer,
             'DayOfMonth':
             load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
         })
예제 #11
0
 def testTrueFalse(self):
     productionrulesetlogical = load_bnf_file(
         "pydsl/contrib/grammar/TrueFalse.bnf")
     parser = BacktracingErrorRecursiveDescentParser(
         productionrulesetlogical)
     tokens = [
         x for x in lex(productionrulesetlogical.alphabet, ascii_encoding,
                        self.tokelist5)
     ]
     result = parser.get_trees(tokens)
     self.assertTrue(result)
예제 #12
0
 def testHTMLTable(self):
     repository = {'integer':RegularExpression("^[0123456789]*$")}
     productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/TrueHTMLTable.bnf", repository)
     parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
     lexed = lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><tr><td>1</td></tr></table>")
     self.assertTrue(lexed)
     result = parser.get_trees(lexed)
     self.assertTrue(result)
     lexed = [x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><td>1</td></tr></table>")]
     result = parser.get_trees(lexed)
     self.assertFalse(result)
예제 #13
0
 def testLogLine(self):
     repository = {'space':String(' '), 
                   'integer':RegularExpression("^[0123456789]*$"),
                   'ipv4':RegularExpression("^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"),
                   'characters':RegularExpression("^[A-z]+$")}
     grammar = load_bnf_file("pydsl/contrib/grammar/logline.bnf", repository)
     checker = checker_factory(grammar)
     original_string = "1.2.3.4 - - [1/1/2003:11:11:11 +2] \"GET\" 1 1 \"referer\" \"useragent\""
     tokenized = lex(grammar.alphabet, ascii_encoding, original_string, force_lexer="general")
     self.assertTrue(checker.check(tokenized))
     self.assertFalse(checker.check("1.2.3.4 - - [1/1/2003:11:11:11 +2] \"GOT\" 1 1 \"referer\" \"useragent\""))
예제 #14
0
 def testEmptyInput(self):
     integer = RegularExpression("^[0123456789]*$")
     date = load_bnf_file(
         "pydsl/contrib/grammar/Date.bnf", {
             'integer':
             integer,
             'DayOfMonth':
             load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
         })
     mydef = Choice([integer, date])
     lexer = lexer_factory(mydef, ascii_encoding)
     self.assertFalse(lexer(""))
예제 #15
0
 def testSimpleLexing(self):
     """Test checker instantiation and call"""
     integer = RegularExpression("^[0123456789]*$")
     date = load_bnf_file(
         "pydsl/contrib/grammar/Date.bnf", {
             'integer':
             integer,
             'DayOfMonth':
             load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
         })
     mydef = Choice([integer, date])
     lexer = lexer_factory(mydef, ascii_encoding)
     self.assertListEqual(lexer("1234"), [Token("1234", integer)])
     self.assertListEqual(lexer([Token(x, ascii_encoding) for x in "1234"]),
                          [Token("1234", integer)])
예제 #16
0
 def testOverlappingLexing(self):
     integer = RegularExpression("^[0123456789]*$")
     date = load_bnf_file(
         "pydsl/contrib/grammar/Date.bnf", {
             'integer':
             integer,
             'DayOfMonth':
             load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
         })
     mydef = Choice([integer, date])
     lexer = lexer_factory(mydef, ascii_encoding)
     self.assertListEqual(lexer("123411/11/2001"), [("1234", integer),
                                                    ("11/11/2001", date)])
     self.assertListEqual(lexer([x for x in "123411/11/2001"]),
                          [("1234", integer), ("11/11/2001", date)])
예제 #17
0
 def testHTMLTable(self):
     repository = {'integer': RegularExpression("^[0123456789]*$")}
     productionrulesetlogical = load_bnf_file(
         "pydsl/contrib/grammar/TrueHTMLTable.bnf", repository)
     parser = BacktracingErrorRecursiveDescentParser(
         productionrulesetlogical)
     lexed = lex(productionrulesetlogical.alphabet, ascii_encoding,
                 "<table><tr><td>1</td></tr></table>")
     self.assertTrue(lexed)
     result = parser.get_trees(lexed)
     self.assertTrue(result)
     lexed = [
         x for x in lex(productionrulesetlogical.alphabet, ascii_encoding,
                        "<table><td>1</td></tr></table>")
     ]
     result = parser.get_trees(lexed)
     self.assertFalse(result)
예제 #18
0
 def setUp(self):
     self.integer = RegularExpression("^[0123456789]*$")
     self.date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':self.integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
예제 #19
0
 def testEmptyInput(self):
     integer = RegularExpression("^[0123456789]*$")
     date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
     mydef = Alphabet([integer,date])
     lexer = lexer_factory(mydef)
     self.assertFalse(lexer(""))
예제 #20
0
 def testFileLoader(self):
     repository = {'integer':RegularExpression("^[0123456789]*$"), 
             'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')}
     self.assertTrue(load_bnf_file("pydsl/contrib/grammar/Date.bnf", repository))
예제 #21
0
 def testTrueFalse(self):
     productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")
     parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
     tokens = [x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, self.tokelist5)]
     result = parser.get_trees(tokens)
     self.assertTrue(result)