def setUp(self):
from pydsl.tree import ParseTree
a = ParseTree(0,6, None, "abcdef")
self.firstleaf1 = ParseTree(0,1, None, "a")
a.append(self.firstleaf1)
b = ParseTree(1,3,None, "bc")
a.append(b)
b.append(ParseTree(1,2,None, "b"))
b.append(ParseTree(2,3,None, "c"))
a.append(ParseTree(3,4,None, "d"))
a.append(ParseTree(4,5,None, "e"))
a.append(ParseTree(5,6,None, "f"))
self.tree1 = a
c = ParseTree(0,6, None, "abcdef")
self.firstleaf2 = ParseTree(0,1, None, "a")
c.append(self.firstleaf2)
b = ParseTree(1,3, None, "bc")
c.append(b)
b.append(ParseTree(1,2, None, "b"))
b.append(ParseTree(2,3, None, "j"))
c.append(ParseTree(3,4, None, "d"))
c.append(ParseTree(4,5, None, "e"))
c.append(ParseTree(5,6, None, "f"))
self.tree2 = c
def __recursive_parser(self,
onlysymbol,
data,
production,
showerrors=False):
""" Aux function. helps check_word"""
LOG.debug("__recursive_parser: Begin ")
if not data:
return []
from pydsl.grammar.symbol import TerminalSymbol, NullSymbol, NonTerminalSymbol
if isinstance(onlysymbol, TerminalSymbol):
LOG.debug("Iteration: terminalsymbol")
return self._reduce_terminal(onlysymbol, data[0], showerrors)
elif isinstance(onlysymbol, NullSymbol):
return [ParseTree(0, 0, onlysymbol, "")]
elif isinstance(onlysymbol, NonTerminalSymbol):
validstack = []
invalidstack = []
for alternative in self._productionset.getProductionsBySide(
onlysymbol): #Alternative
alternativetree = PositionResultList()
alternativeinvalidstack = []
for symbol in alternative.rightside: # Symbol
symbol_success = False
for totalpos in alternativetree.right_limit_list(
): # Right limit
if totalpos >= len(data):
continue
thisresult = self.__recursive_parser(
symbol, data[totalpos:], alternative, showerrors)
if not (thisresult and all(thisresult)):
alternativeinvalidstack += [
x for x in thisresult if not x
]
continue
symbol_success = True
for x in thisresult:
x.shift(totalpos)
success = alternativetree.append(
x.left, x.right, x)
if not success:
#TODO: Add as an error to the tree or to another place
LOG.debug("Discarded symbol :" + str(symbol) +
" position:" + str(totalpos))
else:
LOG.debug("Added symbol :" + str(symbol) +
" position:" + str(totalpos))
if not symbol_success:
LOG.debug("Symbol doesn't work" + str(symbol))
break #Try next alternative
else: # Alternative success (no break happened)
invalidstack += alternativeinvalidstack
for x in alternativetree.valid_sequences():
validstack.append(x)
result = []
LOG.debug("iteration result collection finished:" +
str(validstack))
for alternative in self._productionset.getProductionsBySide(
onlysymbol):
nullcount = alternative.rightside.count(NullSymbol())
for results in validstack:
nnullresults = 0
left = results[0]['left']
right = results[-1]['right']
nnullresults = len([
x for x in results
if x['content'].symbol == NullSymbol()
])
if len(results) - nnullresults != len(
alternative.rightside) - nullcount:
LOG.debug(
"Discarded: incorrect number of non null symbols")
continue
if right > len(data):
LOG.debug("Discarded: length mismatch")
continue
for x in range(
min(len(alternative.rightside), len(results))):
if results[x]['content'] != alternative.rightside[x]:
LOG.debug(
"Discarded: rule doesn't match partial result")
continue
childlist = [x['content'] for x in results]
allvalid = all([x.valid for x in childlist])
if allvalid:
newresult = ParseTree(0,
right - left,
onlysymbol,
data[left:right],
childlist=childlist)
newresult.valid = True
result.append(newresult)
if showerrors and not result:
erroresult = ParseTree(0,
len(data),
onlysymbol,
data,
valid=False)
for invalid in invalidstack:
if invalid.content in production.rightside:
erroresult.append(invalid)
return [erroresult]
return result
raise Exception("Unknown symbol:" + str(onlysymbol))
def __recursive_parser(self, onlysymbol, data, production, showerrors = False):
""" Aux function. helps check_word"""
LOG.debug("__recursive_parser: Begin ")
if not data:
return []
from pydsl.grammar.symbol import TerminalSymbol, NullSymbol, NonTerminalSymbol
if isinstance(onlysymbol, TerminalSymbol):
LOG.debug("Iteration: terminalsymbol")
return self._reduce_terminal(onlysymbol,data[0], showerrors)
elif isinstance(onlysymbol, NullSymbol):
return [ParseTree(0, 0, onlysymbol, "")]
elif isinstance(onlysymbol, NonTerminalSymbol):
validstack = []
invalidstack = []
for alternative in self._productionset.getProductionsBySide(onlysymbol): #Alternative
alternativetree = PositionResultList()
alternativeinvalidstack = []
for symbol in alternative.rightside: # Symbol
symbol_success = False
for totalpos in alternativetree.right_limit_list(): # Right limit
if totalpos >= len(data):
continue
thisresult = self.__recursive_parser(symbol, data[totalpos:], alternative, showerrors)
if not (thisresult and all(thisresult)):
alternativeinvalidstack += [x for x in thisresult if not x]
continue
symbol_success = True
for x in thisresult:
x.shift(totalpos)
success = alternativetree.append(x.left, x.right, x)
if not success:
#TODO: Add as an error to the tree or to another place
LOG.debug("Discarded symbol :" + str(symbol) + " position:" + str(totalpos))
else:
LOG.debug("Added symbol :" + str(symbol) + " position:" + str(totalpos))
if not symbol_success:
LOG.debug("Symbol doesn't work" + str(symbol))
break #Try next alternative
else: # Alternative success (no break happened)
invalidstack += alternativeinvalidstack
for x in alternativetree.valid_sequences():
validstack.append(x)
result = []
LOG.debug("iteration result collection finished:" + str(validstack))
for alternative in self._productionset.getProductionsBySide(onlysymbol):
nullcount = alternative.rightside.count(NullSymbol())
for results in validstack:
nnullresults = 0
left = results[0]['left']
right = results[-1]['right']
nnullresults = len([x for x in results if x['content'].symbol == NullSymbol()])
if len(results) - nnullresults != len(alternative.rightside) - nullcount:
LOG.debug("Discarded: incorrect number of non null symbols")
continue
if right > len(data):
LOG.debug("Discarded: length mismatch")
continue
for x in range(min(len(alternative.rightside), len(results))):
if results[x]['content'] != alternative.rightside[x]:
LOG.debug("Discarded: rule doesn't match partial result")
continue
childlist = [x['content'] for x in results]
allvalid = all([x.valid for x in childlist])
if allvalid:
newresult = ParseTree(0, right - left, onlysymbol,
data[left:right], childlist = childlist)
newresult.valid = True
result.append(newresult)
if showerrors and not result:
erroresult = ParseTree(0,len(data), onlysymbol , data, valid = False)
for invalid in invalidstack:
if invalid.content in production.rightside:
erroresult.append(invalid)
return [erroresult]
return result
raise Exception("Unknown symbol:" + str(onlysymbol))
def setUp(self):
from pydsl.tree import ParseTree
a = ParseTree(0, 6, None, "abcdef")
self.firstleaf1 = ParseTree(0, 1, None, "a")
a.append(self.firstleaf1)
b = ParseTree(1, 3, None, "bc")
a.append(b)
b.append(ParseTree(1, 2, None, "b"))
b.append(ParseTree(2, 3, None, "c"))
a.append(ParseTree(3, 4, None, "d"))
a.append(ParseTree(4, 5, None, "e"))
a.append(ParseTree(5, 6, None, "f"))
self.tree1 = a
c = ParseTree(0, 6, None, "abcdef")
self.firstleaf2 = ParseTree(0, 1, None, "a")
c.append(self.firstleaf2)
b = ParseTree(1, 3, None, "bc")
c.append(b)
b.append(ParseTree(1, 2, None, "b"))
b.append(ParseTree(2, 3, None, "j"))
c.append(ParseTree(3, 4, None, "d"))
c.append(ParseTree(4, 5, None, "e"))
c.append(ParseTree(5, 6, None, "f"))
self.tree2 = c
