def _testParsing(self, st, *L): '''Tests that parsing a particular string all at once will return the given sequence of tokens. The Wait token at the end is implicit.''' # TODO: Test parsing it character-by-character too! #print 'Testing '+repr(st) L = self._collapseText(list(L)) + [WaitToken()] p = Tokenizer() p.queueData(st) resultToks = [] while 1: tok = p.getNextToken() resultToks.append(tok) if isinstance(tok, WaitToken): break resultToks = self._collapseText(resultToks) self.assertEqual(len(L), len(resultToks), "Error parsing '%s': Expected token list %s, got %s" % \ (repr(st), repr(L), repr(resultToks))) for tok,result in zip(L,resultToks): self.assertEqual(tok, result, \ "Error parsing '%s': Expected %s but got %s" % \ (repr(st), repr(L), repr(resultToks))) self.assertEqual(p.getNextToken(), WaitToken(), 'When waiting, repeated calls to getNextToken() should '+\ 'continue to return WaitForMoreDataTokens')
class LineParser(object): """ Assembles tokens into lines composed of TextChunks. This takes care of giving each TextChunk a correct TextStyle based on the flags present in the coloring tokens. """ def __init__(self): self._tokenizer = Tokenizer() self._incompleteLine = [] self._completeLines = [] self._reset() def _reset(self): """ Resets text style info. """ self._style = TextStyle() def getLine(self): """ Gets a line from the tokenizer. This will be None if there isn't one ready, otherwise it will return a list of Chunks. The newline character will not be included in the last chunk. """ if len(self._completeLines) > 0: return self._completeLines.pop(0) else: return None def queueData(self, data): """ Queues a chunk of data and parses it for line retrieval later. """ assert isinstance(data, str), "data must be string" self._tokenizer.queueData(data) # Tokenize now... while 1: tok = self._tokenizer.getNextToken() #print tok.__class__, self._incompleteLine if isinstance(tok, WaitToken): break elif isinstance(tok, TextToken): self._incompleteLine.append(TextChunk(tok.text, self._style)) elif isinstance(tok, AnsiCodeToken): if tok.number == 0: self._reset() elif tok.number == 1: self._style = self._style.clone(bold=True) elif tok.number >= 30 and tok.number <= 37: colorNum = tok.number - 30 self._style = self._style.clone(color=AnsiColor(colorNum)) elif isinstance(tok, TA2Token): flags = tok.getFlags() setting = (flags & ATT_SET)!=0 if flags & ATT_BOLD: self._style = self._style.clone(bold=setting) if flags & ATT_ITALIC: self._style = self._style.clone(italic=setting) if flags & ATT_UNDERLINE: self._style = self._style.clone(underline=setting) if flags & ATT_STRIKETHROUGH: self._style = self._style.clone(strikethrough=setting) if flags & ATT_COLOR: if setting: self._style = \ self._style.clone(color=TA2Color('#%06x'%tok.getColor())) else: self._style = \ self._style.clone(color=None) # TODO: Actually test this # TODO: This will be really slow if someone were to post a line # in Japanese, for example -- every single character # would result in a discrete text chunk! #if flags & ATT_UNICODE: # text = chr(tok.getUnicode()) # #self._incompleteLine.append(TextChunk(text, self._style)) elif isinstance(tok, NewlineToken): self._completeLines.append(self._incompleteLine) self._incompleteLine = [] elif isinstance(tok, XmlToken): self._incompleteLine.append(XmlChunk(tok.xml))