def process(self, stream):
"""Process the stream"""
EOS_TTYPE = T.Whitespace, T.Comment.Single
# Run over all stream tokens
for ttype, value in stream:
# Yield token if we finished a statement and there's no whitespaces
# It will count newline token as a non whitespace. In this context
# whitespace ignores newlines.
# why don't multi line comments also count?
if self.consume_ws and ttype not in EOS_TTYPE:
yield sql.Statement(self.tokens)
# Reset filter and prepare to process next statement
self._reset()
# Change current split level (increase, decrease or remain equal)
self.level += self._change_splitlevel(ttype, value)
# Append the token to the current statement
self.tokens.append(sql.Token(ttype, value))
# Check if we get the end of a statement
if self.level <= 0 and ttype is T.Punctuation and value == ';':
self.consume_ws = True
# Yield pending statement (if any)
if self.tokens and not all(t.is_whitespace for t in self.tokens):
yield sql.Statement(self.tokens)
def process(self, stream):
"""Process the stream"""
EOS_TTYPE = T.Whitespace, T.Comment.Single, T.Comment.Multiline
C_TTYPE = T.Comment.Single, T.Comment.Multiline
# Run over all stream tokens
for params in stream:
pos, ttype, value = params
if self.consume_ws and ttype not in EOS_TTYPE:
yield pos, S.Statement(self.tokens)
self._reset()
self.level += self._change_splitlevel(ttype, value)
self.tokens.append(S.Token(ttype, value))
if self.level <= 0 and ttype is T.Punctuation and value == ';':
self.consume_ws = True
elif ttype in C_TTYPE:
self.consume_ws = True
if self.tokens:
yield pos + len(u''.join([t.value for t in self.tokens])), S.Statement(self.tokens)
