def __init__(self, maxdepth=100): WikiTextTokenizer.__init__(self) self.maxdepth = maxdepth self._tree = self._root = WikiPageTree() self._parse = self._parse_top self._stoptokens = None self._xmlcontext = [] self._stack = [(self._tree, self._parse, self._stoptokens, self._xmlcontext)] return
def handle_text(self, pos, text): WikiTextTokenizer.handle_text(self, pos, text) self.feed_token(pos, text) return
def handle_token(self, pos, token): WikiTextTokenizer.handle_token(self, pos, token) self.feed_token(pos, token) return
def __init__(self): WikiTextTokenizer.__init__(self) self._root = WikiPageTree() self._stack = [(self._root, self._parse_top, set())] (self._tree, self._parse, self._stoptokens) = self._stack.pop() return