def testLogicalExpression(self):
repository = {'TrueFalse':load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")}
productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/LogicalExpression.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "True&&False")]
result = parser.get_trees(tokens)
self.assertTrue(result)
result = parser.get_trees("True&|False")
self.assertFalse(result)
def testLogicalExp(self):
repository = {'TrueFalse':load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")}
productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/LogicalExpression.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
tokens = [x for x in lex(repository['TrueFalse'].alphabet, ascii_encoding, self.tokelist5)]
self.assertEqual(len(tokens), 1)
#tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, Encoding('ascii'), tokens)] #FIXME
tokens = [Token('True', repository['TrueFalse'])]
result = parser.get_trees(tokens)
self.assertTrue(result)
def testLogLine(self):
repository = {
'space':
String(' '),
'integer':
RegularExpression("^[0123456789]*$"),
'ipv4':
RegularExpression(
"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"),
'characters':
RegularExpression("^[A-z]+$")
}
grammar = load_bnf_file("pydsl/contrib/grammar/logline.bnf",
repository)
checker = checker_factory(grammar)
original_string = "1.2.3.4 - - [1/1/2003:11:11:11 +2] \"GET\" 1 1 \"referer\" \"useragent\""
tokenized = lex(grammar.alphabet,
ascii_encoding,
original_string,
force_lexer="general")
self.assertTrue(checker.check(tokenized))
self.assertFalse(
checker.check(
"1.2.3.4 - - [1/1/2003:11:11:11 +2] \"GOT\" 1 1 \"referer\" \"useragent\""
))
def testLogicalExpression(self):
repository = {
'TrueFalse': load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")
}
productionrulesetlogical = load_bnf_file(
"pydsl/contrib/grammar/LogicalExpression.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(
productionrulesetlogical)
tokens = [
x[0] for x in lex(productionrulesetlogical.alphabet,
ascii_encoding, "True&&False")
]
result = parser.get_trees(tokens)
self.assertTrue(result)
result = parser.get_trees("True&|False")
self.assertFalse(result)
def testOverlappingLexing(self):
integer = RegularExpression("^[0123456789]*$")
date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
mydef = Alphabet([integer,date])
lexer = lexer_factory(mydef)
self.assertListEqual(lexer("123411/11/2001"), [("1234", integer),("11/11/2001", date)])
self.assertListEqual(lexer([x for x in "123411/11/2001"]), [("1234", integer),("11/11/2001", date)])
def testLogicalExp(self):
repository = {
'TrueFalse': load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")
}
productionrulesetlogical = load_bnf_file(
"pydsl/contrib/grammar/LogicalExpression.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(
productionrulesetlogical)
tokens = [
x for x in lex(repository['TrueFalse'].alphabet, ascii_encoding,
self.tokelist5)
]
self.assertEqual(len(tokens), 1)
#tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, Encoding('ascii'), tokens)] #FIXME
tokens = [Token('True', repository['TrueFalse'])]
result = parser.get_trees(tokens)
self.assertTrue(result)
def testSimpleLexing(self):
"""Test checker instantiation and call"""
integer = RegularExpression("^[0123456789]*$")
date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
mydef = Choice([integer, date])
lexer = lexer_factory(mydef, ascii_encoding)
self.assertListEqual(lexer("1234"), [Token("1234", integer)])
self.assertListEqual(lexer([Token(x, ascii_encoding) for x in "1234"]), [Token("1234", integer)])
def testSimpleLexing(self):
"""Test checker instantiation and call"""
integer = RegularExpression("^[0123456789]*$")
date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
mydef = Alphabet([integer,date])
lexer = lexer_factory(mydef)
self.assertListEqual(lexer("1234"), [(["1","2","3","4"], integer)])
self.assertListEqual(lexer([x for x in "1234"]), [(["1","2","3","4"], integer)])
def testFileLoader(self):
repository = {
'integer': RegularExpression("^[0123456789]*$"),
'DayOfMonth':
load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
}
self.assertTrue(
load_bnf_file("pydsl/contrib/grammar/Date.bnf", repository))
def setUp(self):
self.integer = RegularExpression("^[0123456789]*$")
self.date = load_bnf_file(
"pydsl/contrib/grammar/Date.bnf", {
'integer':
self.integer,
'DayOfMonth':
load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
})
def testTrueFalse(self):
productionrulesetlogical = load_bnf_file(
"pydsl/contrib/grammar/TrueFalse.bnf")
parser = BacktracingErrorRecursiveDescentParser(
productionrulesetlogical)
tokens = [
x for x in lex(productionrulesetlogical.alphabet, ascii_encoding,
self.tokelist5)
]
result = parser.get_trees(tokens)
self.assertTrue(result)
def testHTMLTable(self):
repository = {'integer':RegularExpression("^[0123456789]*$")}
productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/TrueHTMLTable.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
lexed = lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><tr><td>1</td></tr></table>")
self.assertTrue(lexed)
result = parser.get_trees(lexed)
self.assertTrue(result)
lexed = [x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><td>1</td></tr></table>")]
result = parser.get_trees(lexed)
self.assertFalse(result)
def testLogLine(self):
repository = {'space':String(' '),
'integer':RegularExpression("^[0123456789]*$"),
'ipv4':RegularExpression("^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"),
'characters':RegularExpression("^[A-z]+$")}
grammar = load_bnf_file("pydsl/contrib/grammar/logline.bnf", repository)
checker = checker_factory(grammar)
original_string = "1.2.3.4 - - [1/1/2003:11:11:11 +2] \"GET\" 1 1 \"referer\" \"useragent\""
tokenized = lex(grammar.alphabet, ascii_encoding, original_string, force_lexer="general")
self.assertTrue(checker.check(tokenized))
self.assertFalse(checker.check("1.2.3.4 - - [1/1/2003:11:11:11 +2] \"GOT\" 1 1 \"referer\" \"useragent\""))
def testEmptyInput(self):
integer = RegularExpression("^[0123456789]*$")
date = load_bnf_file(
"pydsl/contrib/grammar/Date.bnf", {
'integer':
integer,
'DayOfMonth':
load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
})
mydef = Choice([integer, date])
lexer = lexer_factory(mydef, ascii_encoding)
self.assertFalse(lexer(""))
def testSimpleLexing(self):
"""Test checker instantiation and call"""
integer = RegularExpression("^[0123456789]*$")
date = load_bnf_file(
"pydsl/contrib/grammar/Date.bnf", {
'integer':
integer,
'DayOfMonth':
load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
})
mydef = Choice([integer, date])
lexer = lexer_factory(mydef, ascii_encoding)
self.assertListEqual(lexer("1234"), [Token("1234", integer)])
self.assertListEqual(lexer([Token(x, ascii_encoding) for x in "1234"]),
[Token("1234", integer)])
def testOverlappingLexing(self):
integer = RegularExpression("^[0123456789]*$")
date = load_bnf_file(
"pydsl/contrib/grammar/Date.bnf", {
'integer':
integer,
'DayOfMonth':
load_python_file('pydsl/contrib/grammar/DayOfMonth.py')
})
mydef = Choice([integer, date])
lexer = lexer_factory(mydef, ascii_encoding)
self.assertListEqual(lexer("123411/11/2001"), [("1234", integer),
("11/11/2001", date)])
self.assertListEqual(lexer([x for x in "123411/11/2001"]),
[("1234", integer), ("11/11/2001", date)])
def testHTMLTable(self):
repository = {'integer': RegularExpression("^[0123456789]*$")}
productionrulesetlogical = load_bnf_file(
"pydsl/contrib/grammar/TrueHTMLTable.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(
productionrulesetlogical)
lexed = lex(productionrulesetlogical.alphabet, ascii_encoding,
"<table><tr><td>1</td></tr></table>")
self.assertTrue(lexed)
result = parser.get_trees(lexed)
self.assertTrue(result)
lexed = [
x for x in lex(productionrulesetlogical.alphabet, ascii_encoding,
"<table><td>1</td></tr></table>")
]
result = parser.get_trees(lexed)
self.assertFalse(result)
def setUp(self):
self.integer = RegularExpression("^[0123456789]*$")
self.date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':self.integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
def testEmptyInput(self):
integer = RegularExpression("^[0123456789]*$")
date = load_bnf_file("pydsl/contrib/grammar/Date.bnf", {'integer':integer, 'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')})
mydef = Alphabet([integer,date])
lexer = lexer_factory(mydef)
self.assertFalse(lexer(""))
def testFileLoader(self):
repository = {'integer':RegularExpression("^[0123456789]*$"),
'DayOfMonth':load_python_file('pydsl/contrib/grammar/DayOfMonth.py')}
self.assertTrue(load_bnf_file("pydsl/contrib/grammar/Date.bnf", repository))
def testTrueFalse(self):
productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")
parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
tokens = [x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, self.tokelist5)]
result = parser.get_trees(tokens)
self.assertTrue(result)
