def setUp(self): from pydsl.tree import ParseTree a = ParseTree(0,6, None, "abcdef") self.firstleaf1 = ParseTree(0,1, None, "a") a.append(self.firstleaf1) b = ParseTree(1,3,None, "bc") a.append(b) b.append(ParseTree(1,2,None, "b")) b.append(ParseTree(2,3,None, "c")) a.append(ParseTree(3,4,None, "d")) a.append(ParseTree(4,5,None, "e")) a.append(ParseTree(5,6,None, "f")) self.tree1 = a c = ParseTree(0,6, None, "abcdef") self.firstleaf2 = ParseTree(0,1, None, "a") c.append(self.firstleaf2) b = ParseTree(1,3, None, "bc") c.append(b) b.append(ParseTree(1,2, None, "b")) b.append(ParseTree(2,3, None, "j")) c.append(ParseTree(3,4, None, "d")) c.append(ParseTree(4,5, None, "e")) c.append(ParseTree(5,6, None, "f")) self.tree2 = c
def __recursive_parser(self, onlysymbol, data, production, showerrors=False): """ Aux function. helps check_word""" LOG.debug("__recursive_parser: Begin ") if not data: return [] from pydsl.grammar.symbol import TerminalSymbol, NullSymbol, NonTerminalSymbol if isinstance(onlysymbol, TerminalSymbol): LOG.debug("Iteration: terminalsymbol") return self._reduce_terminal(onlysymbol, data[0], showerrors) elif isinstance(onlysymbol, NullSymbol): return [ParseTree(0, 0, onlysymbol, "")] elif isinstance(onlysymbol, NonTerminalSymbol): validstack = [] invalidstack = [] for alternative in self._productionset.getProductionsBySide( onlysymbol): #Alternative alternativetree = PositionResultList() alternativeinvalidstack = [] for symbol in alternative.rightside: # Symbol symbol_success = False for totalpos in alternativetree.right_limit_list( ): # Right limit if totalpos >= len(data): continue thisresult = self.__recursive_parser( symbol, data[totalpos:], alternative, showerrors) if not (thisresult and all(thisresult)): alternativeinvalidstack += [ x for x in thisresult if not x ] continue symbol_success = True for x in thisresult: x.shift(totalpos) success = alternativetree.append( x.left, x.right, x) if not success: #TODO: Add as an error to the tree or to another place LOG.debug("Discarded symbol :" + str(symbol) + " position:" + str(totalpos)) else: LOG.debug("Added symbol :" + str(symbol) + " position:" + str(totalpos)) if not symbol_success: LOG.debug("Symbol doesn't work" + str(symbol)) break #Try next alternative else: # Alternative success (no break happened) invalidstack += alternativeinvalidstack for x in alternativetree.valid_sequences(): validstack.append(x) result = [] LOG.debug("iteration result collection finished:" + str(validstack)) for alternative in self._productionset.getProductionsBySide( onlysymbol): nullcount = alternative.rightside.count(NullSymbol()) for results in validstack: nnullresults = 0 left = results[0]['left'] right = results[-1]['right'] nnullresults = len([ x for x in results if x['content'].symbol == NullSymbol() ]) if len(results) - nnullresults != len( alternative.rightside) - nullcount: LOG.debug( "Discarded: incorrect number of non null symbols") continue if right > len(data): LOG.debug("Discarded: length mismatch") continue for x in range( min(len(alternative.rightside), len(results))): if results[x]['content'] != alternative.rightside[x]: LOG.debug( "Discarded: rule doesn't match partial result") continue childlist = [x['content'] for x in results] allvalid = all([x.valid for x in childlist]) if allvalid: newresult = ParseTree(0, right - left, onlysymbol, data[left:right], childlist=childlist) newresult.valid = True result.append(newresult) if showerrors and not result: erroresult = ParseTree(0, len(data), onlysymbol, data, valid=False) for invalid in invalidstack: if invalid.content in production.rightside: erroresult.append(invalid) return [erroresult] return result raise Exception("Unknown symbol:" + str(onlysymbol))
def __recursive_parser(self, onlysymbol, data, production, showerrors = False): """ Aux function. helps check_word""" LOG.debug("__recursive_parser: Begin ") if not data: return [] from pydsl.grammar.symbol import TerminalSymbol, NullSymbol, NonTerminalSymbol if isinstance(onlysymbol, TerminalSymbol): LOG.debug("Iteration: terminalsymbol") return self._reduce_terminal(onlysymbol,data[0], showerrors) elif isinstance(onlysymbol, NullSymbol): return [ParseTree(0, 0, onlysymbol, "")] elif isinstance(onlysymbol, NonTerminalSymbol): validstack = [] invalidstack = [] for alternative in self._productionset.getProductionsBySide(onlysymbol): #Alternative alternativetree = PositionResultList() alternativeinvalidstack = [] for symbol in alternative.rightside: # Symbol symbol_success = False for totalpos in alternativetree.right_limit_list(): # Right limit if totalpos >= len(data): continue thisresult = self.__recursive_parser(symbol, data[totalpos:], alternative, showerrors) if not (thisresult and all(thisresult)): alternativeinvalidstack += [x for x in thisresult if not x] continue symbol_success = True for x in thisresult: x.shift(totalpos) success = alternativetree.append(x.left, x.right, x) if not success: #TODO: Add as an error to the tree or to another place LOG.debug("Discarded symbol :" + str(symbol) + " position:" + str(totalpos)) else: LOG.debug("Added symbol :" + str(symbol) + " position:" + str(totalpos)) if not symbol_success: LOG.debug("Symbol doesn't work" + str(symbol)) break #Try next alternative else: # Alternative success (no break happened) invalidstack += alternativeinvalidstack for x in alternativetree.valid_sequences(): validstack.append(x) result = [] LOG.debug("iteration result collection finished:" + str(validstack)) for alternative in self._productionset.getProductionsBySide(onlysymbol): nullcount = alternative.rightside.count(NullSymbol()) for results in validstack: nnullresults = 0 left = results[0]['left'] right = results[-1]['right'] nnullresults = len([x for x in results if x['content'].symbol == NullSymbol()]) if len(results) - nnullresults != len(alternative.rightside) - nullcount: LOG.debug("Discarded: incorrect number of non null symbols") continue if right > len(data): LOG.debug("Discarded: length mismatch") continue for x in range(min(len(alternative.rightside), len(results))): if results[x]['content'] != alternative.rightside[x]: LOG.debug("Discarded: rule doesn't match partial result") continue childlist = [x['content'] for x in results] allvalid = all([x.valid for x in childlist]) if allvalid: newresult = ParseTree(0, right - left, onlysymbol, data[left:right], childlist = childlist) newresult.valid = True result.append(newresult) if showerrors and not result: erroresult = ParseTree(0,len(data), onlysymbol , data, valid = False) for invalid in invalidstack: if invalid.content in production.rightside: erroresult.append(invalid) return [erroresult] return result raise Exception("Unknown symbol:" + str(onlysymbol))
def setUp(self): from pydsl.tree import ParseTree a = ParseTree(0, 6, None, "abcdef") self.firstleaf1 = ParseTree(0, 1, None, "a") a.append(self.firstleaf1) b = ParseTree(1, 3, None, "bc") a.append(b) b.append(ParseTree(1, 2, None, "b")) b.append(ParseTree(2, 3, None, "c")) a.append(ParseTree(3, 4, None, "d")) a.append(ParseTree(4, 5, None, "e")) a.append(ParseTree(5, 6, None, "f")) self.tree1 = a c = ParseTree(0, 6, None, "abcdef") self.firstleaf2 = ParseTree(0, 1, None, "a") c.append(self.firstleaf2) b = ParseTree(1, 3, None, "bc") c.append(b) b.append(ParseTree(1, 2, None, "b")) b.append(ParseTree(2, 3, None, "j")) c.append(ParseTree(3, 4, None, "d")) c.append(ParseTree(4, 5, None, "e")) c.append(ParseTree(5, 6, None, "f")) self.tree2 = c