def testLogicalExpression(self):
repository = {'TrueFalse':load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")}
productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/LogicalExpression.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "True&&False")]
result = parser.get_trees(tokens)
self.assertTrue(result)
result = parser.get_trees("True&|False")
self.assertFalse(result)
def __init__(self, bnf, parser=None):
Checker.__init__(self)
self.gd = bnf
parser = bnf.options.get("parser", parser)
if parser in ("descent", "auto", "default", None):
from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser
self.__parser = BacktracingErrorRecursiveDescentParser(bnf)
else:
raise ValueError("Unknown parser : " + parser)
def testLogicalExp(self):
repository = {'TrueFalse':load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")}
productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/LogicalExpression.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
tokens = [x for x in lex(repository['TrueFalse'].alphabet, ascii_encoding, self.tokelist5)]
self.assertEqual(len(tokens), 1)
#tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, Encoding('ascii'), tokens)] #FIXME
tokens = [Token('True', repository['TrueFalse'])]
result = parser.get_trees(tokens)
self.assertTrue(result)
def testTrueFalse(self):
productionrulesetlogical = load_bnf_file(
"pydsl/contrib/grammar/TrueFalse.bnf")
parser = BacktracingErrorRecursiveDescentParser(
productionrulesetlogical)
tokens = [
x for x in lex(productionrulesetlogical.alphabet, ascii_encoding,
self.tokelist5)
]
result = parser.get_trees(tokens)
self.assertTrue(result)
def testHTMLTable(self):
repository = {'integer':RegularExpression("^[0123456789]*$")}
productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/TrueHTMLTable.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
lexed = lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><tr><td>1</td></tr></table>")
self.assertTrue(lexed)
result = parser.get_trees(lexed)
self.assertTrue(result)
lexed = [x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, "<table><td>1</td></tr></table>")]
result = parser.get_trees(lexed)
self.assertFalse(result)
def testLogicalExpression(self):
repository = {
'TrueFalse': load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")
}
productionrulesetlogical = load_bnf_file(
"pydsl/contrib/grammar/LogicalExpression.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(
productionrulesetlogical)
tokens = [
x[0] for x in lex(productionrulesetlogical.alphabet,
ascii_encoding, "True&&False")
]
result = parser.get_trees(tokens)
self.assertTrue(result)
result = parser.get_trees("True&|False")
self.assertFalse(result)
class BNFChecker(Checker):
"""Calls another program to perform checking. Args are always file names"""
def __init__(self, bnf, parser = None):
Checker.__init__(self)
self.gd = bnf
parser = bnf.options.get("parser", parser)
if parser in ("descent", "auto", "default", None):
from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser
self.__parser = BacktracingErrorRecursiveDescentParser(bnf)
else:
raise ValueError("Unknown parser : " + parser)
def check(self, data):
if isinstance(data, str):
from pydsl.token import PositionToken
from pydsl.encoding import ascii_encoding
data = [PositionToken(x, ascii_encoding, i, i+1) for i,x in enumerate(data)]
if not isinstance(data, Iterable):
raise TypeError(data)
if not all(check(self.gd.alphabet, [x]) for x in data):
LOG.warning("Invalid input: %s,%s" % (self.gd.alphabet, data))
return False
try:
return len(self.__parser.get_trees(data)) > 0
except IndexError:
return False
class BNFChecker(Checker):
"""Calls another program to perform checking. Args are always file names"""
def __init__(self, bnf, parser=None):
Checker.__init__(self)
self.gd = bnf
parser = bnf.options.get("parser", parser)
if parser in ("descent", "auto", "default", None):
from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser
self.__parser = BacktracingErrorRecursiveDescentParser(bnf)
else:
raise ValueError("Unknown parser : " + parser)
def check(self, data):
if isinstance(data, str):
from pydsl.token import PositionToken
from pydsl.encoding import ascii_encoding
data = [
PositionToken(x, ascii_encoding, i, i + 1)
for i, x in enumerate(data)
]
if not isinstance(data, Iterable):
raise TypeError(data)
if not all(check(self.gd.alphabet, [x]) for x in data):
LOG.warning("Invalid input: %s,%s" % (self.gd.alphabet, data))
return False
try:
return len(self.__parser.get_trees(data)) > 0
except IndexError:
return False
def testLogicalExp(self):
repository = {
'TrueFalse': load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")
}
productionrulesetlogical = load_bnf_file(
"pydsl/contrib/grammar/LogicalExpression.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(
productionrulesetlogical)
tokens = [
x for x in lex(repository['TrueFalse'].alphabet, ascii_encoding,
self.tokelist5)
]
self.assertEqual(len(tokens), 1)
#tokens = [x[0] for x in lex(productionrulesetlogical.alphabet, Encoding('ascii'), tokens)] #FIXME
tokens = [Token('True', repository['TrueFalse'])]
result = parser.get_trees(tokens)
self.assertTrue(result)
def testHTMLTable(self):
repository = {'integer': RegularExpression("^[0123456789]*$")}
productionrulesetlogical = load_bnf_file(
"pydsl/contrib/grammar/TrueHTMLTable.bnf", repository)
parser = BacktracingErrorRecursiveDescentParser(
productionrulesetlogical)
lexed = lex(productionrulesetlogical.alphabet, ascii_encoding,
"<table><tr><td>1</td></tr></table>")
self.assertTrue(lexed)
result = parser.get_trees(lexed)
self.assertTrue(result)
lexed = [
x for x in lex(productionrulesetlogical.alphabet, ascii_encoding,
"<table><td>1</td></tr></table>")
]
result = parser.get_trees(lexed)
self.assertFalse(result)
def testRecursiveDescentParserNullBad(self):
descentparser = BacktracingErrorRecursiveDescentParser(productionset2)
from pydsl.encoding import ascii_encoding
ascii_encoding = ascii_encoding
lexed_string4 = lex(productionset2.alphabet, ascii_encoding, string4)
result = descentparser(lexed_string4)
self.assertFalse(result)
result = descentparser(list(string4))
self.assertFalse(result)
def __init__(self, bnf, parser = None):
Checker.__init__(self)
self.gd = bnf
parser = bnf.options.get("parser", parser)
if parser in ("descent", "auto", "default", None):
from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser
self.__parser = BacktracingErrorRecursiveDescentParser(bnf)
else:
raise ValueError("Unknown parser : " + parser)
def test_main_case(self):
input_data = "1+2"
ascii_lexer = lexer_factory(ascii_encoding, None)
ascii_tokens = [x for x in ascii_lexer(input_data)]
self.assertListEqual([str(x) for x in ascii_tokens], ['1', '+', '2'])
def concept_translator_fun(inputtokens):
result = []
for x in inputtokens:
if str(x) == "1":
result.append("one")
elif str(x) == "2":
result.append("two")
elif str(x) == "+":
result.append("addition")
else:
raise Exception(x.__class__.__name__)
return result
def to_number(number):
if number == "one":
return 1
if number == "two":
return 2
math_expression_concepts = concept_translator_fun(ascii_tokens)
self.assertListEqual(math_expression_concepts,
['one', 'addition', 'two'])
grammar_def = [
"S ::= E",
"E ::= one addition two",
"one := String,one",
"two := String,two",
"addition := String,addition",
]
from pydsl.file.BNF import strlist_to_production_set
production_set = strlist_to_production_set(grammar_def, {})
from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser
rdp = BacktracingErrorRecursiveDescentParser(production_set)
parse_tree = rdp(math_expression_concepts)
from pydsl.grammar.symbol import NonTerminalSymbol
def parse_tree_walker(tree):
if tree.symbol == NonTerminalSymbol("S"):
return parse_tree_walker(tree.childlist[0])
if tree.symbol == NonTerminalSymbol("E"):
return to_number(tree.childlist[0].symbol.gd) + to_number(
tree.childlist[2].symbol.gd)
raise Exception
result = parse_tree_walker(parse_tree[0])
self.assertEqual(result, 3)
def parser_factory(grammar, parser = None):
from pydsl.grammar.BNF import BNFGrammar
if isinstance(grammar, BNFGrammar):
if parser in ("auto" , "default" , "descent", None):
from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser
return BacktracingErrorRecursiveDescentParser(grammar)
elif parser == "lr0":
from pydsl.parser.LR0 import LR0Parser
return LR0Parser(grammar)
elif parser == "ll1":
from pydsl.parser.LL import LL1RecursiveDescentParser
return LL1RecursiveDescentParser(grammar)
else:
raise Exception("Wrong parser name: " + str(parser))
else:
raise ValueError(grammar)
class BNFChecker(Checker):
"""Calls another program to perform checking. Args are always file names"""
def __init__(self, bnf, parser = None):
Checker.__init__(self)
self.gd = bnf
parser = bnf.options.get("parser",parser)
if parser in ("descent", "auto", "default", None):
from pydsl.parser.backtracing import BacktracingErrorRecursiveDescentParser
self.__parser = BacktracingErrorRecursiveDescentParser(bnf)
else:
raise ValueError("Unknown parser : " + parser)
def check(self, data):
for element in data:
if not check(self.gd.alphabet, element):
LOG.warning("Invalid input: %s,%s" % (self.gd.alphabet, element))
return False
try:
return len(self.__parser.get_trees(data)) > 0
except IndexError:
return False
def testRecursiveLeftRecursion(self):
descentparser = BacktracingErrorRecursiveDescentParser(productionsetlr)
self.assertRaises(RuntimeError, descentparser, dots)
def testRecursiveDescentParserStore(self):
descentparser = BacktracingErrorRecursiveDescentParser(productionset1)
result = descentparser(string1)
self.assertTrue(result)
result = descentparser(list(string1))
self.assertTrue(result)
def testRecursiveDescentParserBad(self):
descentparser = BacktracingErrorRecursiveDescentParser(productionset1)
result = descentparser(string2)
self.assertFalse(result)
result = descentparser(list(string2))
self.assertFalse(result)
def testRecursiveDescentParserNull(self):
descentparser = BacktracingErrorRecursiveDescentParser(productionset2)
result = descentparser(string3)
self.assertTrue(result)
result = descentparser(list(string3))
self.assertTrue(result)
def testTrueFalse(self):
productionrulesetlogical = load_bnf_file("pydsl/contrib/grammar/TrueFalse.bnf")
parser = BacktracingErrorRecursiveDescentParser(productionrulesetlogical)
tokens = [x for x in lex(productionrulesetlogical.alphabet, ascii_encoding, self.tokelist5)]
result = parser.get_trees(tokens)
self.assertTrue(result)
def testCenterRecursion(self):
descentparser = BacktracingErrorRecursiveDescentParser(productionsetcr)
result = descentparser(dots)
self.assertTrue(result)
result = descentparser(list(dots))
self.assertTrue(result)
