def _reduce_terminal(self, symbol, data, showerrors = False):
from pydsl.check import check
from pydsl.tree import ParseTree
result = check(symbol.gd, [data])
if result:
return [ParseTree(0,1, symbol , data)]
if showerrors and not result:
return [ParseTree(0,1, symbol , data, valid = False)]
return []
def __aux_parser(self, symbol):
from pydsl.grammar.symbol import TerminalSymbol
if isinstance(symbol, TerminalSymbol):
LOG.debug("matching symbol %s, data:%s, index:%s" %
(symbol, self.data, self.index))
result = self.match(symbol)
LOG.debug("symbol matched %s" % result)
return result
productions = self._productionset.getProductionsBySide(symbol)
valid_firsts = []
for production in productions:
first_of_production = self._productionset.first_lookup(
production.rightside[0])
if check(first_of_production, [self.current]):
valid_firsts.append(production)
if len(valid_firsts) != 1:
raise ParseError(
"Expected only one valid production, found %s" %
len(valid_firsts), 0)
childlist = [self.__aux_parser(x) for x in valid_firsts[0].rightside]
left = childlist[0].left
right = childlist[-1].right
content = [x.content for x in childlist]
return ParseTree(left, right, symbol, content, childlist=childlist)
def setUp(self):
from pydsl.tree import ParseTree
a = ParseTree(0,6, None, "abcdef")
self.firstleaf1 = ParseTree(0,1, None, "a")
a.append(self.firstleaf1)
b = ParseTree(1,3,None, "bc")
a.append(b)
b.append(ParseTree(1,2,None, "b"))
b.append(ParseTree(2,3,None, "c"))
a.append(ParseTree(3,4,None, "d"))
a.append(ParseTree(4,5,None, "e"))
a.append(ParseTree(5,6,None, "f"))
self.tree1 = a
c = ParseTree(0,6, None, "abcdef")
self.firstleaf2 = ParseTree(0,1, None, "a")
c.append(self.firstleaf2)
b = ParseTree(1,3, None, "bc")
c.append(b)
b.append(ParseTree(1,2, None, "b"))
b.append(ParseTree(2,3, None, "j"))
c.append(ParseTree(3,4, None, "d"))
c.append(ParseTree(4,5, None, "e"))
c.append(ParseTree(5,6, None, "f"))
self.tree2 = c
def __recursive_parser(self, onlysymbol, data, production, showerrors = False):
""" Aux function. helps check_word"""
LOG.debug("__recursive_parser: Begin ")
if not data:
return []
from pydsl.grammar.symbol import TerminalSymbol, NullSymbol, NonTerminalSymbol
if isinstance(onlysymbol, TerminalSymbol):
LOG.debug("Iteration: terminalsymbol")
return self._reduce_terminal(onlysymbol,data[0], showerrors)
elif isinstance(onlysymbol, NullSymbol):
return [ParseTree(0, 0, onlysymbol, "")]
elif isinstance(onlysymbol, NonTerminalSymbol):
validstack = []
invalidstack = []
for alternative in self._productionset.getProductionsBySide(onlysymbol): #Alternative
alternativetree = PositionResultList()
alternativeinvalidstack = []
for symbol in alternative.rightside: # Symbol
symbol_success = False
for totalpos in alternativetree.right_limit_list(): # Right limit
if totalpos >= len(data):
continue
thisresult = self.__recursive_parser(symbol, data[totalpos:], alternative, showerrors)
if not (thisresult and all(thisresult)):
alternativeinvalidstack += [x for x in thisresult if not x]
continue
symbol_success = True
for x in thisresult:
x.shift(totalpos)
success = alternativetree.append(x.left, x.right, x)
if not success:
#TODO: Add as an error to the tree or to another place
LOG.debug("Discarded symbol :" + str(symbol) + " position:" + str(totalpos))
else:
LOG.debug("Added symbol :" + str(symbol) + " position:" + str(totalpos))
if not symbol_success:
LOG.debug("Symbol doesn't work" + str(symbol))
break #Try next alternative
else: # Alternative success (no break happened)
invalidstack += alternativeinvalidstack
for x in alternativetree.valid_sequences():
validstack.append(x)
result = []
LOG.debug("iteration result collection finished:" + str(validstack))
for alternative in self._productionset.getProductionsBySide(onlysymbol):
nullcount = alternative.rightside.count(NullSymbol())
for results in validstack:
nnullresults = 0
left = results[0]['left']
right = results[-1]['right']
nnullresults = len([x for x in results if x['content'].symbol == NullSymbol()])
if len(results) - nnullresults != len(alternative.rightside) - nullcount:
LOG.debug("Discarded: incorrect number of non null symbols")
continue
if right > len(data):
LOG.debug("Discarded: length mismatch")
continue
for x in range(min(len(alternative.rightside), len(results))):
if results[x]['content'] != alternative.rightside[x]:
LOG.debug("Discarded: rule doesn't match partial result")
continue
childlist = [x['content'] for x in results]
allvalid = all([x.valid for x in childlist])
if allvalid:
newresult = ParseTree(0, right - left, onlysymbol,
data[left:right], childlist = childlist)
newresult.valid = True
result.append(newresult)
if showerrors and not result:
erroresult = ParseTree(0,len(data), onlysymbol , data, valid = False)
for invalid in invalidstack:
if invalid.content in production.rightside:
erroresult.append(invalid)
return [erroresult]
return result
raise Exception("Unknown symbol:" + str(onlysymbol))
def __recursive_parser(self,
onlysymbol,
data,
production,
showerrors=False):
""" Aux function. helps check_word"""
LOG.debug("__recursive_parser: Begin ")
if not data:
return []
from pydsl.grammar.symbol import TerminalSymbol, NullSymbol, NonTerminalSymbol
if isinstance(onlysymbol, TerminalSymbol):
LOG.debug("Iteration: terminalsymbol")
return self._reduce_terminal(onlysymbol, data[0], showerrors)
elif isinstance(onlysymbol, NullSymbol):
return [ParseTree(0, 0, onlysymbol, "")]
elif isinstance(onlysymbol, NonTerminalSymbol):
validstack = []
invalidstack = []
for alternative in self._productionset.getProductionsBySide(
onlysymbol): #Alternative
alternativetree = PositionResultList()
alternativeinvalidstack = []
for symbol in alternative.rightside: # Symbol
symbol_success = False
for totalpos in alternativetree.right_limit_list(
): # Right limit
if totalpos >= len(data):
continue
thisresult = self.__recursive_parser(
symbol, data[totalpos:], alternative, showerrors)
if not (thisresult and all(thisresult)):
alternativeinvalidstack += [
x for x in thisresult if not x
]
continue
symbol_success = True
for x in thisresult:
x.shift(totalpos)
success = alternativetree.append(
x.left, x.right, x)
if not success:
#TODO: Add as an error to the tree or to another place
LOG.debug("Discarded symbol :" + str(symbol) +
" position:" + str(totalpos))
else:
LOG.debug("Added symbol :" + str(symbol) +
" position:" + str(totalpos))
if not symbol_success:
LOG.debug("Symbol doesn't work" + str(symbol))
break #Try next alternative
else: # Alternative success (no break happened)
invalidstack += alternativeinvalidstack
for x in alternativetree.valid_sequences():
validstack.append(x)
result = []
LOG.debug("iteration result collection finished:" +
str(validstack))
for alternative in self._productionset.getProductionsBySide(
onlysymbol):
nullcount = alternative.rightside.count(NullSymbol())
for results in validstack:
nnullresults = 0
left = results[0]['left']
right = results[-1]['right']
nnullresults = len([
x for x in results
if x['content'].symbol == NullSymbol()
])
if len(results) - nnullresults != len(
alternative.rightside) - nullcount:
LOG.debug(
"Discarded: incorrect number of non null symbols")
continue
if right > len(data):
LOG.debug("Discarded: length mismatch")
continue
for x in range(
min(len(alternative.rightside), len(results))):
if results[x]['content'] != alternative.rightside[x]:
LOG.debug(
"Discarded: rule doesn't match partial result")
continue
childlist = [x['content'] for x in results]
allvalid = all([x.valid for x in childlist])
if allvalid:
newresult = ParseTree(0,
right - left,
onlysymbol,
data[left:right],
childlist=childlist)
newresult.valid = True
result.append(newresult)
if showerrors and not result:
erroresult = ParseTree(0,
len(data),
onlysymbol,
data,
valid=False)
for invalid in invalidstack:
if invalid.content in production.rightside:
erroresult.append(invalid)
return [erroresult]
return result
raise Exception("Unknown symbol:" + str(onlysymbol))
def setUp(self):
from pydsl.tree import ParseTree
a = ParseTree(0, 6, None, "abcdef")
self.firstleaf1 = ParseTree(0, 1, None, "a")
a.append(self.firstleaf1)
b = ParseTree(1, 3, None, "bc")
a.append(b)
b.append(ParseTree(1, 2, None, "b"))
b.append(ParseTree(2, 3, None, "c"))
a.append(ParseTree(3, 4, None, "d"))
a.append(ParseTree(4, 5, None, "e"))
a.append(ParseTree(5, 6, None, "f"))
self.tree1 = a
c = ParseTree(0, 6, None, "abcdef")
self.firstleaf2 = ParseTree(0, 1, None, "a")
c.append(self.firstleaf2)
b = ParseTree(1, 3, None, "bc")
c.append(b)
b.append(ParseTree(1, 2, None, "b"))
b.append(ParseTree(2, 3, None, "j"))
c.append(ParseTree(3, 4, None, "d"))
c.append(ParseTree(4, 5, None, "e"))
c.append(ParseTree(5, 6, None, "f"))
self.tree2 = c
def match(self, symbol):
if symbol.check([self.current]):
current = self.current
self.consume()
return ParseTree(self.index - 1, self.index, symbol, current)
raise Exception("Not matched")
