def process(self, stack, stream): splitlevel = 0 stmt = None consume_ws = False stmt_tokens = [] for ttype, value in stream: # Before appending the token if (consume_ws and ttype is not T.Whitespace and ttype is not T.Comment.Single): consume_ws = False stmt.tokens = stmt_tokens yield stmt self._reset() stmt = None splitlevel = 0 if stmt is None: stmt = Statement() stmt_tokens = [] splitlevel += self._change_splitlevel(ttype, value) # Append the token stmt_tokens.append(Token(ttype, value)) # After appending the token if (splitlevel <= 0 and ttype is T.Punctuation and value == ';'): consume_ws = True if stmt is not None: stmt.tokens = stmt_tokens yield stmt
def process(self, stack, stream): "Process the stream" consume_ws = False splitlevel = 0 stmt = None stmt_tokens = [] # Run over all stream tokens for ttype, value in stream: # Yield token if we finished a statement and there's no whitespaces if consume_ws and ttype not in (T.Whitespace, T.Comment.Single): stmt.tokens = stmt_tokens yield stmt # Reset filter and prepare to process next statement self._reset() consume_ws = False splitlevel = 0 stmt = None # Create a new statement if we are not currently in one of them if stmt is None: stmt = Statement() stmt_tokens = [] # Change current split level (increase, decrease or remain equal) splitlevel += self._change_splitlevel(ttype, value) # Append the token to the current statement stmt_tokens.append(Token(ttype, value)) # Check if we get the end of a statement if splitlevel <= 0 and ttype is T.Punctuation and value == ';': consume_ws = True # Yield pending statement (if any) if stmt is not None: stmt.tokens = stmt_tokens yield stmt