def _reduce_terminal(self, symbol, data, showerrors = False): from pydsl.check import check from pydsl.tree import ParseTree result = check(symbol.gd, [data]) if result: return [ParseTree(0,1, symbol , data)] if showerrors and not result: return [ParseTree(0,1, symbol , data, valid = False)] return []
def __aux_parser(self, symbol): from pydsl.grammar.symbol import TerminalSymbol if isinstance(symbol, TerminalSymbol): LOG.debug("matching symbol %s, data:%s, index:%s" % (symbol, self.data, self.index)) result = self.match(symbol) LOG.debug("symbol matched %s" % result) return result productions = self._productionset.getProductionsBySide(symbol) valid_firsts = [] for production in productions: first_of_production = self._productionset.first_lookup( production.rightside[0]) if check(first_of_production, [self.current]): valid_firsts.append(production) if len(valid_firsts) != 1: raise ParseError( "Expected only one valid production, found %s" % len(valid_firsts), 0) childlist = [self.__aux_parser(x) for x in valid_firsts[0].rightside] left = childlist[0].left right = childlist[-1].right content = [x.content for x in childlist] return ParseTree(left, right, symbol, content, childlist=childlist)
def setUp(self): from pydsl.tree import ParseTree a = ParseTree(0,6, None, "abcdef") self.firstleaf1 = ParseTree(0,1, None, "a") a.append(self.firstleaf1) b = ParseTree(1,3,None, "bc") a.append(b) b.append(ParseTree(1,2,None, "b")) b.append(ParseTree(2,3,None, "c")) a.append(ParseTree(3,4,None, "d")) a.append(ParseTree(4,5,None, "e")) a.append(ParseTree(5,6,None, "f")) self.tree1 = a c = ParseTree(0,6, None, "abcdef") self.firstleaf2 = ParseTree(0,1, None, "a") c.append(self.firstleaf2) b = ParseTree(1,3, None, "bc") c.append(b) b.append(ParseTree(1,2, None, "b")) b.append(ParseTree(2,3, None, "j")) c.append(ParseTree(3,4, None, "d")) c.append(ParseTree(4,5, None, "e")) c.append(ParseTree(5,6, None, "f")) self.tree2 = c
def __recursive_parser(self, onlysymbol, data, production, showerrors = False): """ Aux function. helps check_word""" LOG.debug("__recursive_parser: Begin ") if not data: return [] from pydsl.grammar.symbol import TerminalSymbol, NullSymbol, NonTerminalSymbol if isinstance(onlysymbol, TerminalSymbol): LOG.debug("Iteration: terminalsymbol") return self._reduce_terminal(onlysymbol,data[0], showerrors) elif isinstance(onlysymbol, NullSymbol): return [ParseTree(0, 0, onlysymbol, "")] elif isinstance(onlysymbol, NonTerminalSymbol): validstack = [] invalidstack = [] for alternative in self._productionset.getProductionsBySide(onlysymbol): #Alternative alternativetree = PositionResultList() alternativeinvalidstack = [] for symbol in alternative.rightside: # Symbol symbol_success = False for totalpos in alternativetree.right_limit_list(): # Right limit if totalpos >= len(data): continue thisresult = self.__recursive_parser(symbol, data[totalpos:], alternative, showerrors) if not (thisresult and all(thisresult)): alternativeinvalidstack += [x for x in thisresult if not x] continue symbol_success = True for x in thisresult: x.shift(totalpos) success = alternativetree.append(x.left, x.right, x) if not success: #TODO: Add as an error to the tree or to another place LOG.debug("Discarded symbol :" + str(symbol) + " position:" + str(totalpos)) else: LOG.debug("Added symbol :" + str(symbol) + " position:" + str(totalpos)) if not symbol_success: LOG.debug("Symbol doesn't work" + str(symbol)) break #Try next alternative else: # Alternative success (no break happened) invalidstack += alternativeinvalidstack for x in alternativetree.valid_sequences(): validstack.append(x) result = [] LOG.debug("iteration result collection finished:" + str(validstack)) for alternative in self._productionset.getProductionsBySide(onlysymbol): nullcount = alternative.rightside.count(NullSymbol()) for results in validstack: nnullresults = 0 left = results[0]['left'] right = results[-1]['right'] nnullresults = len([x for x in results if x['content'].symbol == NullSymbol()]) if len(results) - nnullresults != len(alternative.rightside) - nullcount: LOG.debug("Discarded: incorrect number of non null symbols") continue if right > len(data): LOG.debug("Discarded: length mismatch") continue for x in range(min(len(alternative.rightside), len(results))): if results[x]['content'] != alternative.rightside[x]: LOG.debug("Discarded: rule doesn't match partial result") continue childlist = [x['content'] for x in results] allvalid = all([x.valid for x in childlist]) if allvalid: newresult = ParseTree(0, right - left, onlysymbol, data[left:right], childlist = childlist) newresult.valid = True result.append(newresult) if showerrors and not result: erroresult = ParseTree(0,len(data), onlysymbol , data, valid = False) for invalid in invalidstack: if invalid.content in production.rightside: erroresult.append(invalid) return [erroresult] return result raise Exception("Unknown symbol:" + str(onlysymbol))
def __recursive_parser(self, onlysymbol, data, production, showerrors=False): """ Aux function. helps check_word""" LOG.debug("__recursive_parser: Begin ") if not data: return [] from pydsl.grammar.symbol import TerminalSymbol, NullSymbol, NonTerminalSymbol if isinstance(onlysymbol, TerminalSymbol): LOG.debug("Iteration: terminalsymbol") return self._reduce_terminal(onlysymbol, data[0], showerrors) elif isinstance(onlysymbol, NullSymbol): return [ParseTree(0, 0, onlysymbol, "")] elif isinstance(onlysymbol, NonTerminalSymbol): validstack = [] invalidstack = [] for alternative in self._productionset.getProductionsBySide( onlysymbol): #Alternative alternativetree = PositionResultList() alternativeinvalidstack = [] for symbol in alternative.rightside: # Symbol symbol_success = False for totalpos in alternativetree.right_limit_list( ): # Right limit if totalpos >= len(data): continue thisresult = self.__recursive_parser( symbol, data[totalpos:], alternative, showerrors) if not (thisresult and all(thisresult)): alternativeinvalidstack += [ x for x in thisresult if not x ] continue symbol_success = True for x in thisresult: x.shift(totalpos) success = alternativetree.append( x.left, x.right, x) if not success: #TODO: Add as an error to the tree or to another place LOG.debug("Discarded symbol :" + str(symbol) + " position:" + str(totalpos)) else: LOG.debug("Added symbol :" + str(symbol) + " position:" + str(totalpos)) if not symbol_success: LOG.debug("Symbol doesn't work" + str(symbol)) break #Try next alternative else: # Alternative success (no break happened) invalidstack += alternativeinvalidstack for x in alternativetree.valid_sequences(): validstack.append(x) result = [] LOG.debug("iteration result collection finished:" + str(validstack)) for alternative in self._productionset.getProductionsBySide( onlysymbol): nullcount = alternative.rightside.count(NullSymbol()) for results in validstack: nnullresults = 0 left = results[0]['left'] right = results[-1]['right'] nnullresults = len([ x for x in results if x['content'].symbol == NullSymbol() ]) if len(results) - nnullresults != len( alternative.rightside) - nullcount: LOG.debug( "Discarded: incorrect number of non null symbols") continue if right > len(data): LOG.debug("Discarded: length mismatch") continue for x in range( min(len(alternative.rightside), len(results))): if results[x]['content'] != alternative.rightside[x]: LOG.debug( "Discarded: rule doesn't match partial result") continue childlist = [x['content'] for x in results] allvalid = all([x.valid for x in childlist]) if allvalid: newresult = ParseTree(0, right - left, onlysymbol, data[left:right], childlist=childlist) newresult.valid = True result.append(newresult) if showerrors and not result: erroresult = ParseTree(0, len(data), onlysymbol, data, valid=False) for invalid in invalidstack: if invalid.content in production.rightside: erroresult.append(invalid) return [erroresult] return result raise Exception("Unknown symbol:" + str(onlysymbol))
def setUp(self): from pydsl.tree import ParseTree a = ParseTree(0, 6, None, "abcdef") self.firstleaf1 = ParseTree(0, 1, None, "a") a.append(self.firstleaf1) b = ParseTree(1, 3, None, "bc") a.append(b) b.append(ParseTree(1, 2, None, "b")) b.append(ParseTree(2, 3, None, "c")) a.append(ParseTree(3, 4, None, "d")) a.append(ParseTree(4, 5, None, "e")) a.append(ParseTree(5, 6, None, "f")) self.tree1 = a c = ParseTree(0, 6, None, "abcdef") self.firstleaf2 = ParseTree(0, 1, None, "a") c.append(self.firstleaf2) b = ParseTree(1, 3, None, "bc") c.append(b) b.append(ParseTree(1, 2, None, "b")) b.append(ParseTree(2, 3, None, "j")) c.append(ParseTree(3, 4, None, "d")) c.append(ParseTree(4, 5, None, "e")) c.append(ParseTree(5, 6, None, "f")) self.tree2 = c
def match(self, symbol): if symbol.check([self.current]): current = self.current self.consume() return ParseTree(self.index - 1, self.index, symbol, current) raise Exception("Not matched")