def process(self, stream): """Process the stream""" EOS_TTYPE = T.Whitespace, T.Comment.Single # Run over all stream tokens for ttype, value in stream: # Yield token if we finished a statement and there's no whitespaces # It will count newline token as a non whitespace. In this context # whitespace ignores newlines. # why don't multi line comments also count? if self.consume_ws and ttype not in EOS_TTYPE: yield sql.Statement(self.tokens) # Reset filter and prepare to process next statement self._reset() # Change current split level (increase, decrease or remain equal) self.level += self._change_splitlevel(ttype, value) # Append the token to the current statement self.tokens.append(sql.Token(ttype, value)) # Check if we get the end of a statement if self.level <= 0 and ttype is T.Punctuation and value == ';': self.consume_ws = True # Yield pending statement (if any) if self.tokens and not all(t.is_whitespace for t in self.tokens): yield sql.Statement(self.tokens)
def process(self, stream): """Process the stream""" EOS_TTYPE = T.Whitespace, T.Comment.Single, T.Comment.Multiline C_TTYPE = T.Comment.Single, T.Comment.Multiline # Run over all stream tokens for params in stream: pos, ttype, value = params if self.consume_ws and ttype not in EOS_TTYPE: yield pos, S.Statement(self.tokens) self._reset() self.level += self._change_splitlevel(ttype, value) self.tokens.append(S.Token(ttype, value)) if self.level <= 0 and ttype is T.Punctuation and value == ';': self.consume_ws = True elif ttype in C_TTYPE: self.consume_ws = True if self.tokens: yield pos + len(u''.join([t.value for t in self.tokens])), S.Statement(self.tokens)